diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index faca52ad053bc..d51f802eceeef 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -7,7 +7,7 @@ https://github.com/pingcap/docs/tree/master/resources/doc-templates ### First-time contributors' checklist -- [ ] I've signed [**Contributor License Agreement**](https://cla-assistant.io/pingcap/docs) that's required for repo owners to accept my contribution. +- [ ] I've signed the [**Contributor License Agreement**](https://cla.pingcap.net/pingcap/docs), which is required for the repository owners to accept my contribution. ### What is changed, added or deleted? (Required) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1a93b303192fd..9e3777e34106a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,8 +11,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + uses: actions/checkout@v6 + - uses: actions/setup-node@v6 with: node-version: "18" - name: Verify duplicated file names @@ -32,8 +32,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + uses: actions/checkout@v6 + - uses: actions/setup-node@v6 with: node-version: "18" - name: Check TOC-tidb-cloud.md existence @@ -57,16 +57,16 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v6 - name: Vale Linter - uses: errata-ai/vale-action@v2.0.1 + uses: errata-ai/vale-action@reviewdog with: # Optional # Specify file path to lint #files: . # Optional - onlyAnnotateModifiedLines: true + filter_mode: added env: # Required, set by GitHub actions automatically: # https://docs.github.com/en/actions/security-guides/automatic-token-authentication#about-the-github_token-secret diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 35c5683680f50..05a0d487b07ec 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -6,6 +6,7 @@ on: - ".github/**" branches: - master + - release-7.5 - release-7.2 - release-7.1 - release-7.0 diff --git a/.github/workflows/media.yml b/.github/workflows/media.yml index 71ae0a603ac6b..fbe6cb6f20136 100644 --- a/.github/workflows/media.yml +++ b/.github/workflows/media.yml @@ -7,11 +7,11 @@ on: paths: - media/** jobs: - run: + upload: name: Upload media files runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 with: # Must use at least depth 2! fetch-depth: 2 @@ -34,3 +34,42 @@ jobs: # printf "%s\n" ${{ secrets.AWS_ACCESS_KEY }} ${{ secrets.AWS_SECRET_KEY }} ${{ secrets.AWS_REGION }} "json" | aws configure - name: Upload run: cloud-assets-utils verify-and-sync -qiniu true -qiniu-bucket ${{ secrets.QINIU_BUCKET_NAME }} media -replace-first-path-to images/docs -cdn-refresh https://download.pingcap.com/ + + - name: Install coscli + run: | + wget https://cosbrowser.cloud.tencent.com/software/coscli/coscli-linux-amd64 + mv coscli-linux-amd64 coscli + chmod 755 coscli + + - name: Upload to COS + run: | + ./coscli sync media/ cos://${{ secrets.TENCENTCLOUD_BUCKET_ID }}/media/images/docs \ + --init-skip \ + --recursive \ + --routines 16 \ + --secret-id ${{ secrets.TENCENTCLOUD_SECRET_ID }} \ + --secret-key ${{ secrets.TENCENTCLOUD_SECRET_KEY }} \ + --endpoint cos.ap-beijing.myqcloud.com + + cdn-refresh: + needs: upload + runs-on: ubuntu-latest + name: Refresh CDN Cache + env: + TENCENTCLOUD_SECRET_ID: ${{ secrets.TENCENTCLOUD_SECRET_ID }} + TENCENTCLOUD_SECRET_KEY: ${{ secrets.TENCENTCLOUD_SECRET_KEY }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Python environment + uses: actions/setup-python@v6 + with: + python-version: "3.12" + architecture: "x64" + + - name: Install Tencent Cloud CLI + run: pipx install tccli + + - name: Purge production CDN cache + run: tccli cdn PurgePathCache --Paths '["https://docs-download.pingcap.com/media/images/docs/"]' --FlushType delete diff --git a/.github/workflows/prevent-deletion.yaml b/.github/workflows/prevent-deletion.yaml index 62b6c32c9a506..9d08804908a40 100644 --- a/.github/workflows/prevent-deletion.yaml +++ b/.github/workflows/prevent-deletion.yaml @@ -15,32 +15,44 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout base - uses: actions/checkout@v3 + uses: actions/checkout@v6 + with: + fetch-depth: 0 - name: Fetch head + env: + HEAD_CLONE_URL: ${{ github.event.pull_request.head.repo.clone_url }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} run: | - git remote add head ${{ github.event.pull_request.head.repo.clone_url }} - git fetch --depth=1 head ${{ github.event.pull_request.head.ref }} + git remote add head "$HEAD_CLONE_URL" + git fetch head -- "$HEAD_REF" - name: Find changes + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - git rev-parse '${{ github.event.pull_request.head.sha }}' - if git diff --merge-base --name-only --diff-filter 'D' HEAD '${{ github.event.pull_request.head.sha }}' | grep -E '^media/.*\.(jpg|png|jpeg|gif)$' >/tmp/changed_files; then + git rev-parse "$HEAD_SHA" + if git diff --merge-base --name-only --diff-filter 'D' HEAD "$HEAD_SHA" | grep -E '^media/.*\.(jpg|png|jpeg|gif)$' >/tmp/changed_files; then cat /tmp/changed_files - echo '{"name":"Image Deletion Check","head_sha":"${{ github.event.pull_request.head.sha }}","status":"completed","conclusion":"failure"}' > /tmp/body.json + jq -n --arg sha "$HEAD_SHA" \ + '{name:"Image Deletion Check",head_sha:$sha,status:"completed",conclusion:"failure"}' > /tmp/body.json jq \ --arg count "$(wc -l /tmp/changed_files | awk '{print $1}')" \ --arg summary "$(cat /tmp/changed_files | sed 's/^/- /')" \ '.output.title = "Found " + $count + " deleted images" | .output.summary = $summary' \ /tmp/body.json > /tmp/body2.json else - echo '{"name":"Image Deletion Check","head_sha":"${{ github.event.pull_request.head.sha }}","status":"completed","conclusion":"success","output":{"title":"OK","summary":"No deleted images"}}' > /tmp/body2.json + jq -n --arg sha "$HEAD_SHA" \ + '{name:"Image Deletion Check",head_sha:$sha,status:"completed",conclusion:"success",output:{title:"OK",summary:"No deleted images"}}' > /tmp/body2.json fi - name: Publish result + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} run: | cat /tmp/body2.json curl \ -sSL \ -X POST \ -H "Accept: application/vnd.github+json" \ - -H "Authorization: token ${{ github.token }}" \ + -H "Authorization: token $GH_TOKEN" \ -T '/tmp/body2.json' \ - 'https://api.github.com/repos/${{ github.repository }}/check-runs' + "https://api.github.com/repos/$REPO/check-runs" diff --git a/.github/workflows/rebase.yml b/.github/workflows/rebase.yml index 73cce3c0fa894..3db5ea50b7172 100644 --- a/.github/workflows/rebase.yml +++ b/.github/workflows/rebase.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout the latest code - uses: actions/checkout@v3 + uses: actions/checkout@v6 with: token: ${{ secrets.REBASE_SECRET_KEY }} fetch-depth: 0 # otherwise, you will fail to push refs to dest repo diff --git a/.lycheeignore b/.lycheeignore index 3ead57f29a342..4ad628acc44ef 100644 --- a/.lycheeignore +++ b/.lycheeignore @@ -14,4 +14,23 @@ file://.*?http:/\$%7BPD_IP%7D:\$%7BPD_PORT%7D/dashboard.* http://\{grafana-ip\}:3000 http://\{pd-ip\}:2379/dashboard http://localhost:\d+/ -https://github\.com/\$user/(docs|docs-cn) \ No newline at end of file +https://github\.com/\$user/(docs|docs-cn) +https://linux\.die\.net/man.* +https://dev\.mysql\.com/doc/.+/5.7/en/.* +https://dev\.mysql\.com/doc/.+/8\.0/en/.* +https://dev\.mysql\.com/doc/.+/8\.4/en/.* +https://dev\.mysql\.com/doc/[a-z\-]+/en/.* +https://dev\.mysql\.com/doc/relnotes/[a-z\-]+/en/.* +https://dev\.mysql\.com/doc/dev/mysql-server/.* +https://dev\.mysql\.com/downloads/.* +https://bugs\.mysql\.com/bug\.php.* +https://www\.mysql\.com/products/.* +https://platform\.openai\.com/docs/.* +https://openai\.com/.* +https://jwt\.io/ +https://typeorm\.io/.* +https://dash\.cloudflare\.com/.* +https://centminmod\.com/mydumper\.html +https://docs\.pingcap\.com/tidb/v6\.6/system-variables#tidb_pessimistic_txn_aggressive_locking-new-in-v660 +https://docs\.pingcap\.com/tidb/v7\.6/system-variables#tidb_ddl_version-new-in-v760 +https://developers\.redhat\.com/blog/2021/01/05/building-red-hat-enterprise-linux-9-for-the-x86-64-v2-microarchitecture-level diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3bb3edc102c5d..c522c5cfc0c57 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,7 +96,7 @@ Please perform the following steps to create your Pull Request to this repositor ### Step 0: Sign the CLA -Your Pull Requests can only be merged after you sign the [Contributor License Agreement](https://cla-assistant.io/pingcap/docs) (CLA). Please make sure you sign the CLA before continuing. +To have your pull requests merged, you must sign the [Contributor License Agreement](https://cla.pingcap.net/pingcap/docs) (CLA). Please make sure you sign it before continuing. ### Step 1: Fork the repository diff --git a/OWNERS b/OWNERS index ec95e7e30cb68..928908987c355 100644 --- a/OWNERS +++ b/OWNERS @@ -10,6 +10,7 @@ approvers: - dragonly - en-jin19 - hfxsd + - Icemap - jackysp - kissmydb - lance6716 @@ -40,7 +41,6 @@ reviewers: - ericsyh - glkappe - GMHDBJD - - Icemap - Joyinqin - junlan-zhang - KanShiori diff --git a/README.md b/README.md index 47aa3d173b871..a54701d2fffba 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Currently, we maintain the following versions of TiDB documentation in different | Branch name | TiDB docs version | | :---------|:----------| | [`master`](https://github.com/pingcap/docs/tree/master) | The latest development version | +| [`release-7.5`](https://github.com/pingcap/docs/tree/release-7.5) | 7.5 LTS (Long-Term Support) | | [`release-7.4`](https://github.com/pingcap/docs/tree/release-7.4) | 7.4 Development Milestone Release | | [`release-7.3`](https://github.com/pingcap/docs/tree/release-7.3) | 7.3 Development Milestone Release | | [`release-7.2`](https://github.com/pingcap/docs/tree/release-7.2) | 7.2 Development Milestone Release | diff --git a/TOC-tidb-cloud.md b/TOC-tidb-cloud.md index f44ecfee672c1..dc3010b726af7 100644 --- a/TOC-tidb-cloud.md +++ b/TOC-tidb-cloud.md @@ -3,27 +3,34 @@ - [Docs Home](https://docs.pingcap.com/) - About TiDB Cloud - - [Why TiDB Cloud](/tidb-cloud/tidb-cloud-intro.md) + - [What is TiDB Cloud](/tidb-cloud/tidb-cloud-intro.md) - [Architecture](/tidb-cloud/tidb-cloud-intro.md#architecture) - - [High Availability](/tidb-cloud/high-availability-with-multi-az.md) + - High Availability + - [Multi-AZ Deployments](/tidb-cloud/high-availability-with-multi-az.md) + - [High Availability in TiDB Cloud Serverless](/tidb-cloud/serverless-high-availability.md) - [MySQL Compatibility](/mysql-compatibility.md) - [Roadmap](/tidb-cloud/tidb-cloud-roadmap.md) - Get Started - [Try Out TiDB Cloud](/tidb-cloud/tidb-cloud-quickstart.md) + - [Try Out TiDB + AI](/tidb-cloud/vector-search-get-started-using-python.md) - [Try Out HTAP](/tidb-cloud/tidb-cloud-htap-quickstart.md) - [Try Out TiDB Cloud CLI](/tidb-cloud/get-started-with-cli.md) - [Perform a PoC](/tidb-cloud/tidb-cloud-poc.md) - Develop Applications - [Overview](/develop/dev-guide-overview.md) - Quick Start - - [Build a TiDB Serverless Cluster in TiDB Cloud](/develop/dev-guide-build-cluster-in-cloud.md) + - [Build a TiDB Cloud Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md) - [CRUD SQL in TiDB](/develop/dev-guide-tidb-crud-sql.md) - Connect to TiDB Cloud - GUI Database Tools - [JetBrains DataGrip](/develop/dev-guide-gui-datagrip.md) - [DBeaver](/develop/dev-guide-gui-dbeaver.md) - [VS Code](/develop/dev-guide-gui-vscode-sqltools.md) + - [MySQL Workbench](/develop/dev-guide-gui-mysql-workbench.md) + - [Navicat](/develop/dev-guide-gui-navicat.md) - [Choose Driver or ORM](/develop/dev-guide-choose-driver-or-orm.md) + - BI + - [Looker Studio](/tidb-cloud/dev-guide-bi-looker-studio.md) - Java - [JDBC](/develop/dev-guide-sample-application-java-jdbc.md) - [MyBatis](/develop/dev-guide-sample-application-java-mybatis.md) @@ -51,6 +58,13 @@ - Ruby - [mysql2](/develop/dev-guide-sample-application-ruby-mysql2.md) - [Rails](/develop/dev-guide-sample-application-ruby-rails.md) + - [WordPress](/tidb-cloud/dev-guide-wordpress.md) + - Serverless Driver (Beta) + - [TiDB Cloud Serverless Driver](/tidb-cloud/serverless-driver.md) + - [Node.js Example](/tidb-cloud/serverless-driver-node-example.md) + - [Prisma Example](/tidb-cloud/serverless-driver-prisma-example.md) + - [Kysely Example](/tidb-cloud/serverless-driver-kysely-example.md) + - [Drizzle Example](/tidb-cloud/serverless-driver-drizzle-example.md) - Third-Party Support - [Third-Party Tools Supported by TiDB](/develop/dev-guide-third-party-support.md) - [Known Incompatibility Issues with Third-Party Tools](/develop/dev-guide-third-party-tools-compatibility.md) @@ -104,10 +118,10 @@ - [Select Your Cluster Tier](/tidb-cloud/select-cluster-tier.md) - [Determine Your TiDB Size](/tidb-cloud/size-your-cluster.md) - [TiDB Cloud Performance Reference](/tidb-cloud/tidb-cloud-performance-reference.md) - - Manage TiDB Serverless Clusters - - [Create a TiDB Serverless Cluster](/tidb-cloud/create-tidb-cluster-serverless.md) - - Connect to Your TiDB Serverless Cluster - - [Connection Method Overview](/tidb-cloud/connect-to-tidb-cluster-serverless.md) + - Manage TiDB Cloud Serverless Clusters + - [Create a TiDB Cloud Serverless Cluster](/tidb-cloud/create-tidb-cluster-serverless.md) + - Connect to Your TiDB Cloud Serverless Cluster + - [Connection Overview](/tidb-cloud/connect-to-tidb-cluster-serverless.md) - [Connect via Public Endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md) - [Connect via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md) - Branch (Beta) @@ -115,19 +129,20 @@ - [Manage Branches](/tidb-cloud/branch-manage.md) - [GitHub Integration](/tidb-cloud/branch-github-integration.md) - [Manage Spending Limit](/tidb-cloud/manage-serverless-spend-limit.md) - - [Back Up and Restore TiDB Serverless Data](/tidb-cloud/backup-and-restore-serverless.md) - - Manage TiDB Dedicated Clusters - - [Create a TiDB Dedicated Cluster](/tidb-cloud/create-tidb-cluster.md) - - Connect to Your TiDB Dedicated Cluster + - [Back Up and Restore TiDB Cloud Serverless Data](/tidb-cloud/backup-and-restore-serverless.md) + - [Export Data from TiDB Cloud Serverless](/tidb-cloud/serverless-export.md) + - Manage TiDB Cloud Dedicated Clusters + - [Create a TiDB Cloud Dedicated Cluster](/tidb-cloud/create-tidb-cluster.md) + - Connect to Your TiDB Cloud Dedicated Cluster - [Connection Method Overview](/tidb-cloud/connect-to-tidb-cluster.md) - - [Connect via Standard Connection](/tidb-cloud/connect-via-standard-connection.md) + - [Connect via Public Connection](/tidb-cloud/connect-via-standard-connection.md) - [Connect via Private Endpoint with AWS](/tidb-cloud/set-up-private-endpoint-connections.md) - [Connect via Private Endpoint (Private Service Connect) with Google Cloud](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md) - [Connect via VPC Peering](/tidb-cloud/set-up-vpc-peering-connections.md) - [Connect via SQL Shell](/tidb-cloud/connect-via-sql-shell.md) - - [Scale a TiDB Dedicated Cluster](/tidb-cloud/scale-tidb-cluster.md) - - [Back Up and Restore TiDB Dedicated Data](/tidb-cloud/backup-and-restore.md) - - [Pause or Resume a TiDB Dedicated Cluster](/tidb-cloud/pause-or-resume-tidb-cluster.md) + - [Scale a TiDB Cloud Dedicated Cluster](/tidb-cloud/scale-tidb-cluster.md) + - [Back Up and Restore TiDB Cloud Dedicated Data](/tidb-cloud/backup-and-restore.md) + - [Pause or Resume a TiDB Cloud Dedicated Cluster](/tidb-cloud/pause-or-resume-tidb-cluster.md) - [Configure Maintenance Window](/tidb-cloud/configure-maintenance-window.md) - Use an HTAP Cluster with TiFlash - [TiFlash Overview](/tiflash/tiflash-overview.md) @@ -139,12 +154,18 @@ - [TiFlash Query Result Materialization](/tiflash/tiflash-results-materialization.md) - [TiFlash Late Materialization](/tiflash/tiflash-late-materialization.md) - [Compatibility](/tiflash/tiflash-compatibility.md) + - [Pipeline Execution Model](/tiflash/tiflash-pipeline-model.md) - Monitor and Alert - [Overview](/tidb-cloud/monitor-tidb-cluster.md) - [Built-in Metrics](/tidb-cloud/built-in-monitoring.md) - [Built-in Alerting](/tidb-cloud/monitor-built-in-alerting.md) + - Subscribe to Alert Notifications + - [Subscribe via Email](/tidb-cloud/monitor-alert-email.md) + - [Subscribe via Slack](/tidb-cloud/monitor-alert-slack.md) + - [Subscribe via Zoom](/tidb-cloud/monitor-alert-zoom.md) - [Cluster Events](/tidb-cloud/tidb-cloud-events.md) - [Third-Party Metrics Integrations (Beta)](/tidb-cloud/third-party-monitoring-integrations.md) + - [TiDB Cloud Clinic](/tidb-cloud/tidb-cloud-clinic.md) - Tune Performance - [Overview](/tidb-cloud/tidb-cloud-tune-performance-overview.md) - Analyze Performance @@ -216,21 +237,53 @@ - [Import Sample Data (SQL File)](/tidb-cloud/import-sample-data.md) - [Import CSV Files from Amazon S3 or GCS](/tidb-cloud/import-csv-files.md) - [Import Apache Parquet Files from Amazon S3 or GCS](/tidb-cloud/import-parquet-files.md) + - [Import with MySQL CLI](/tidb-cloud/import-with-mysql-cli.md) - Reference - - [Configure Amazon S3 Access and GCS Access](/tidb-cloud/config-s3-and-gcs-access.md) + - [Configure External Storage Access for TiDB Dedicated](/tidb-cloud/config-s3-and-gcs-access.md) + - [Configure External Storage Access for TiDB Serverless](/tidb-cloud/serverless-external-storage.md) - [Naming Conventions for Data Import](/tidb-cloud/naming-conventions-for-data-import.md) - [CSV Configurations for Importing Data](/tidb-cloud/csv-config-for-import-data.md) - [Troubleshoot Access Denied Errors during Data Import from Amazon S3](/tidb-cloud/troubleshoot-import-access-denied-error.md) - [Precheck Errors, Migration Errors, and Alerts for Data Migration](/tidb-cloud/tidb-cloud-dm-precheck-and-troubleshooting.md) + - [Connect AWS DMS to TiDB Cloud clusters](/tidb-cloud/tidb-cloud-connect-aws-dms.md) - Explore Data - - [Chat2Query (Beta)](/tidb-cloud/explore-data-with-chat2query.md) + - [Chat2Query (Beta) in SQL Editor](/tidb-cloud/explore-data-with-chat2query.md) + - [SQL Proxy Account](/tidb-cloud/sql-proxy-account.md) +- Vector Search (Beta) + - [Overview](/tidb-cloud/vector-search-overview.md) + - Get Started + - [Get Started with SQL](/tidb-cloud/vector-search-get-started-using-sql.md) + - [Get Started with Python](/tidb-cloud/vector-search-get-started-using-python.md) + - Integrations + - [Overview](/tidb-cloud/vector-search-integration-overview.md) + - AI Frameworks + - [LlamaIndex](/tidb-cloud/vector-search-integrate-with-llamaindex.md) + - [Langchain](/tidb-cloud/vector-search-integrate-with-langchain.md) + - Embedding Models/Services + - [Jina AI](/tidb-cloud/vector-search-integrate-with-jinaai-embedding.md) + - ORM Libraries + - [SQLAlchemy](/tidb-cloud/vector-search-integrate-with-sqlalchemy.md) + - [peewee](/tidb-cloud/vector-search-integrate-with-peewee.md) + - [Django ORM](/tidb-cloud/vector-search-integrate-with-django-orm.md) + - Reference + - [Vector Data Types](/tidb-cloud/vector-search-data-types.md) + - [Vector Functions and Operators](/tidb-cloud/vector-search-functions-and-operators.md) + - [Vector Index](/tidb-cloud/vector-search-index.md) + - [Improve Performance](/tidb-cloud/vector-search-improve-performance.md) + - [Limitations](/tidb-cloud/vector-search-limitations.md) + - [Changelogs](/tidb-cloud/vector-search-changelogs.md) - Data Service (Beta) - [Overview](/tidb-cloud/data-service-overview.md) - [Get Started](/tidb-cloud/data-service-get-started.md) - - [Try Out Chat2Query API](/tidb-cloud/use-chat2query-api.md) + - Chat2Query API + - [Get Started](/tidb-cloud/use-chat2query-api.md) + - [Start Multi-round Chat2Query](/tidb-cloud/use-chat2query-sessions.md) + - [Use Knowledge Bases](/tidb-cloud/use-chat2query-knowledge.md) - [Manage Data App](/tidb-cloud/data-service-manage-data-app.md) - [Manage Endpoint](/tidb-cloud/data-service-manage-endpoint.md) - [API Key](/tidb-cloud/data-service-api-key.md) + - [Custom Domain](/tidb-cloud/data-service-custom-domain.md) + - [Integrations](/tidb-cloud/data-service-integrations.md) - [Run in Postman](/tidb-cloud/data-service-postman-integration.md) - [Deploy Automatically with GitHub](/tidb-cloud/data-service-manage-github-connection.md) - [Use OpenAPI Specification with Next.js](/tidb-cloud/data-service-oas-with-nextjs.md) @@ -239,41 +292,66 @@ - Stream Data - [Changefeed Overview](/tidb-cloud/changefeed-overview.md) - [To MySQL Sink](/tidb-cloud/changefeed-sink-to-mysql.md) - - [To Kafka Sink (Beta)](/tidb-cloud/changefeed-sink-to-apache-kafka.md) + - [To Kafka Sink](/tidb-cloud/changefeed-sink-to-apache-kafka.md) - [To TiDB Cloud Sink](/tidb-cloud/changefeed-sink-to-tidb-cloud.md) - [To Cloud Storage](/tidb-cloud/changefeed-sink-to-cloud-storage.md) + - Reference + - [Set Up Self-Hosted Kafka Private Link Service in AWS](/tidb-cloud/setup-self-hosted-kafka-private-link-service.md) + - [Set Up Self-Hosted Kafka Private Service Connect in Google Cloud](/tidb-cloud/setup-self-hosted-kafka-private-service-connect.md) +- Disaster Recovery + - [Recovery Group Overview](/tidb-cloud/recovery-group-overview.md) + - [Get Started](/tidb-cloud/recovery-group-get-started.md) + - [Failover and Reprotect Databases](/tidb-cloud/recovery-group-failover.md) + - [Delete a Recovery Group](/tidb-cloud/recovery-group-delete.md) - Security - Identity Access Control - [Password Authentication](/tidb-cloud/tidb-cloud-password-authentication.md) - - [SSO Authentication](/tidb-cloud/tidb-cloud-sso-authentication.md) + - [Basic SSO Authentication](/tidb-cloud/tidb-cloud-sso-authentication.md) + - [Organization SSO Authentication](/tidb-cloud/tidb-cloud-org-sso-authentication.md) - [Identity Access Management](/tidb-cloud/manage-user-access.md) + - [OAuth 2.0](/tidb-cloud/oauth2.md) - Network Access Control - - TiDB Serverless + - TiDB Cloud Serverless - [Connect via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md) - - [TLS Connections to TiDB Serverless](/tidb-cloud/secure-connections-to-serverless-clusters.md) - - TiDB Dedicated + - [TLS Connections to TiDB Cloud Serverless](/tidb-cloud/secure-connections-to-serverless-clusters.md) + - TiDB Cloud Dedicated - [Configure an IP Access List](/tidb-cloud/configure-ip-access-list.md) - - [Connect via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md) + - [Connect via Private Endpoint with AWS](/tidb-cloud/set-up-private-endpoint-connections.md) + - [Connect via Private Endpoint (Private Service Connect) with Google Cloud](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md) - [Connect via VPC Peering](/tidb-cloud/set-up-vpc-peering-connections.md) - - [TLS Connections to TiDB Dedicated](/tidb-cloud/tidb-cloud-tls-connect-to-dedicated.md) + - [TLS Connections to TiDB Cloud Dedicated](/tidb-cloud/tidb-cloud-tls-connect-to-dedicated.md) - Data Access Control - [Encryption at Rest Using Customer-Managed Encryption Keys](/tidb-cloud/tidb-cloud-encrypt-cmek.md) - Database Access Control - - [Configure Cluster Security Settings](/tidb-cloud/configure-security-settings.md) + - [Configure Cluster Password Settings](/tidb-cloud/configure-security-settings.md) - Audit Management - [Database Audit Logging](/tidb-cloud/tidb-cloud-auditing.md) - [Console Audit Logging](/tidb-cloud/tidb-cloud-console-auditing.md) - Billing - [Invoices](/tidb-cloud/tidb-cloud-billing.md#invoices) - [Billing Details](/tidb-cloud/tidb-cloud-billing.md#billing-details) + - [Cost Explorer](/tidb-cloud/tidb-cloud-billing.md#cost-explorer) + - [Billing Profile](/tidb-cloud/tidb-cloud-billing.md#billing-profile) - [Credits](/tidb-cloud/tidb-cloud-billing.md#credits) - [Payment Method Setting](/tidb-cloud/tidb-cloud-billing.md#payment-method) - [Billing from AWS or GCP Marketplace](/tidb-cloud/tidb-cloud-billing.md#billing-from-aws-marketplace-or-google-cloud-marketplace) - [Billing for Changefeed](/tidb-cloud/tidb-cloud-billing-ticdc-rcu.md) - [Billing for Data Migration](/tidb-cloud/tidb-cloud-billing-dm.md) + - [Billing for Recovery Groups](/tidb-cloud/tidb-cloud-billing-recovery-group.md) + - [Manage Budgets](/tidb-cloud/tidb-cloud-budget.md) +- TiDB Cloud Partner Web Console + - [TiDB Cloud Partners](/tidb-cloud/tidb-cloud-partners.md) + - [MSP Customer](/tidb-cloud/managed-service-provider-customer.md) + - [Reseller's Customer](/tidb-cloud/cppo-customer.md) - API - [API Overview](/tidb-cloud/api-overview.md) - - [API Reference](https://docs.pingcap.com/tidbcloud/api/v1beta) + - API Reference + - v1beta1 + - [Billing](https://docs.pingcap.com/tidbcloud/api/v1beta1/billing) + - [Data Service](https://docs.pingcap.com/tidbcloud/api/v1beta1/dataservice) + - [IAM](https://docs.pingcap.com/tidbcloud/api/v1beta1/iam) + - [MSP (Deprecated)](https://docs.pingcap.com/tidbcloud/api/v1beta1/msp) + - [v1beta](https://docs.pingcap.com/tidbcloud/api/v1beta) - Integrations - [Airbyte](/tidb-cloud/integrate-tidbcloud-with-airbyte.md) - [Amazon AppFlow](/develop/dev-guide-aws-appflow-integration.md) @@ -302,13 +380,26 @@ - [Computing](/tidb-computing.md) - [Scheduling](/tidb-scheduling.md) - [TSO](/tso.md) - - [TiDB Dedicated Limitations and Quotas](/tidb-cloud/limitations-and-quotas.md) - - [TiDB Serverless Limitations](/tidb-cloud/serverless-limitations.md) + - [TiDB Cloud Dedicated Limitations and Quotas](/tidb-cloud/limitations-and-quotas.md) + - [TiDB Cloud Serverless Limitations](/tidb-cloud/serverless-limitations.md) - [Limited SQL Features on TiDB Cloud](/tidb-cloud/limited-sql-features.md) - [TiDB Limitations](/tidb-limitations.md) + - TiDB Distributed eXecution Framework (DXF) + - [Introduction](/tidb-distributed-execution-framework.md) + - [TiDB Global Sort](/tidb-global-sort.md) - Benchmarks - - [TPC-C Performance Test Report](/tidb-cloud/v7.1.0-performance-benchmarking-with-tpcc.md) - - [Sysbench Performance Test Report](/tidb-cloud/v7.1.0-performance-benchmarking-with-sysbench.md) + - TiDB v8.1 + - [TPC-C Performance Test Report](/tidb-cloud/v8.1-performance-benchmarking-with-tpcc.md) + - [Sysbench Performance Test Report](/tidb-cloud/v8.1-performance-benchmarking-with-sysbench.md) + - TiDB v7.5 + - [TPC-C Performance Test Report](/tidb-cloud/v7.5-performance-benchmarking-with-tpcc.md) + - [Sysbench Performance Test Report](/tidb-cloud/v7.5-performance-benchmarking-with-sysbench.md) + - TiDB v7.1 + - [TPC-C Performance Test Report](/tidb-cloud/v7.1-performance-benchmarking-with-tpcc.md) + - [Sysbench Performance Test Report](/tidb-cloud/v7.1-performance-benchmarking-with-sysbench.md) + - TiDB v6.5 + - [TPC-C Performance Test Report](/tidb-cloud/v6.5-performance-benchmarking-with-tpcc.md) + - [Sysbench Performance Test Report](/tidb-cloud/v6.5-performance-benchmarking-with-sysbench.md) - SQL - [Explore SQL with TiDB](/basic-sql-operations.md) - SQL Language Structure and Syntax @@ -323,8 +414,6 @@ - [Expression Syntax](/expression-syntax.md) - [Comment Syntax](/comment-syntax.md) - SQL Statements - - [`ADD COLUMN`](/sql-statements/sql-statement-add-column.md) - - [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) - [`ADMIN`](/sql-statements/sql-statement-admin.md) - [`ADMIN CANCEL DDL`](/sql-statements/sql-statement-admin-cancel-ddl.md) - [`ADMIN CHECKSUM TABLE`](/sql-statements/sql-statement-admin-checksum-table.md) @@ -335,19 +424,28 @@ - [`ADMIN RESUME DDL`](/sql-statements/sql-statement-admin-resume-ddl.md) - [`ADMIN SHOW DDL [JOBS|JOB QUERIES]`](/sql-statements/sql-statement-admin-show-ddl.md) - [`ALTER DATABASE`](/sql-statements/sql-statement-alter-database.md) - - [`ALTER INDEX`](/sql-statements/sql-statement-alter-index.md) - [`ALTER INSTANCE`](/sql-statements/sql-statement-alter-instance.md) - [`ALTER PLACEMENT POLICY`](/sql-statements/sql-statement-alter-placement-policy.md) - [`ALTER RANGE`](/sql-statements/sql-statement-alter-range.md) - [`ALTER RESOURCE GROUP`](/sql-statements/sql-statement-alter-resource-group.md) - - [`ALTER TABLE`](/sql-statements/sql-statement-alter-table.md) - - [`ALTER TABLE COMPACT`](/sql-statements/sql-statement-alter-table-compact.md) + - [`ALTER SEQUENCE`](/sql-statements/sql-statement-alter-sequence.md) + - `ALTER TABLE` + - [Overview](/sql-statements/sql-statement-alter-table.md) + - [`ADD COLUMN`](/sql-statements/sql-statement-add-column.md) + - [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) + - [`ALTER INDEX`](/sql-statements/sql-statement-alter-index.md) + - [`CHANGE COLUMN`](/sql-statements/sql-statement-change-column.md) + - [`COMPACT`](/sql-statements/sql-statement-alter-table-compact.md) + - [`DROP COLUMN`](/sql-statements/sql-statement-drop-column.md) + - [`DROP INDEX`](/sql-statements/sql-statement-drop-index.md) + - [`MODIFY COLUMN`](/sql-statements/sql-statement-modify-column.md) + - [`RENAME INDEX`](/sql-statements/sql-statement-rename-index.md) - [`ALTER USER`](/sql-statements/sql-statement-alter-user.md) - [`ANALYZE TABLE`](/sql-statements/sql-statement-analyze-table.md) - [`BACKUP`](/sql-statements/sql-statement-backup.md) - [`BATCH`](/sql-statements/sql-statement-batch.md) - [`BEGIN`](/sql-statements/sql-statement-begin.md) - - [`CHANGE COLUMN`](/sql-statements/sql-statement-change-column.md) + - [`CANCEL IMPORT JOB`](/sql-statements/sql-statement-cancel-import-job.md) - [`COMMIT`](/sql-statements/sql-statement-commit.md) - [`CREATE [GLOBAL|SESSION] BINDING`](/sql-statements/sql-statement-create-binding.md) - [`CREATE DATABASE`](/sql-statements/sql-statement-create-database.md) @@ -366,7 +464,6 @@ - [`DESCRIBE`](/sql-statements/sql-statement-describe.md) - [`DO`](/sql-statements/sql-statement-do.md) - [`DROP [GLOBAL|SESSION] BINDING`](/sql-statements/sql-statement-drop-binding.md) - - [`DROP COLUMN`](/sql-statements/sql-statement-drop-column.md) - [`DROP DATABASE`](/sql-statements/sql-statement-drop-database.md) - [`DROP INDEX`](/sql-statements/sql-statement-drop-index.md) - [`DROP PLACEMENT POLICY`](/sql-statements/sql-statement-drop-placement-policy.md) @@ -380,7 +477,7 @@ - [`EXECUTE`](/sql-statements/sql-statement-execute.md) - [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) - [`EXPLAIN`](/sql-statements/sql-statement-explain.md) - - [`FLASHBACK CLUSTER TO TIMESTAMP`](/sql-statements/sql-statement-flashback-to-timestamp.md) + - [`FLASHBACK CLUSTER`](/sql-statements/sql-statement-flashback-cluster.md) - [`FLASHBACK DATABASE`](/sql-statements/sql-statement-flashback-database.md) - [`FLASHBACK TABLE`](/sql-statements/sql-statement-flashback-table.md) - [`FLUSH PRIVILEGES`](/sql-statements/sql-statement-flush-privileges.md) @@ -388,17 +485,16 @@ - [`FLUSH TABLES`](/sql-statements/sql-statement-flush-tables.md) - [`GRANT `](/sql-statements/sql-statement-grant-privileges.md) - [`GRANT `](/sql-statements/sql-statement-grant-role.md) + - [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) - [`INSERT`](/sql-statements/sql-statement-insert.md) - [`KILL [TIDB]`](/sql-statements/sql-statement-kill.md) - [`LOAD DATA`](/sql-statements/sql-statement-load-data.md) - [`LOAD STATS`](/sql-statements/sql-statement-load-stats.md) - [`LOCK STATS`](/sql-statements/sql-statement-lock-stats.md) - [`LOCK TABLES` and `UNLOCK TABLES`](/sql-statements/sql-statement-lock-tables-and-unlock-tables.md) - - [`MODIFY COLUMN`](/sql-statements/sql-statement-modify-column.md) - [`PREPARE`](/sql-statements/sql-statement-prepare.md) - [`QUERY WATCH`](/sql-statements/sql-statement-query-watch.md) - [`RECOVER TABLE`](/sql-statements/sql-statement-recover-table.md) - - [`RENAME INDEX`](/sql-statements/sql-statement-rename-index.md) - [`RENAME TABLE`](/sql-statements/sql-statement-rename-table.md) - [`RENAME USER`](/sql-statements/sql-statement-rename-user.md) - [`REPLACE`](/sql-statements/sql-statement-replace.md) @@ -433,16 +529,15 @@ - [`SHOW ERRORS`](/sql-statements/sql-statement-show-errors.md) - [`SHOW [FULL] FIELDS FROM`](/sql-statements/sql-statement-show-fields-from.md) - [`SHOW GRANTS`](/sql-statements/sql-statement-show-grants.md) - - [`SHOW INDEX [FROM|IN]`](/sql-statements/sql-statement-show-index.md) + - [`SHOW IMPORT JOB`](/sql-statements/sql-statement-show-import-job.md) - [`SHOW INDEXES [FROM|IN]`](/sql-statements/sql-statement-show-indexes.md) - - [`SHOW KEYS [FROM|IN]`](/sql-statements/sql-statement-show-keys.md) - [`SHOW MASTER STATUS`](/sql-statements/sql-statement-show-master-status.md) - [`SHOW PLACEMENT`](/sql-statements/sql-statement-show-placement.md) - [`SHOW PLACEMENT FOR`](/sql-statements/sql-statement-show-placement-for.md) - [`SHOW PLACEMENT LABELS`](/sql-statements/sql-statement-show-placement-labels.md) - [`SHOW PLUGINS`](/sql-statements/sql-statement-show-plugins.md) - [`SHOW PRIVILEGES`](/sql-statements/sql-statement-show-privileges.md) - - [`SHOW [FULL] PROCESSSLIST`](/sql-statements/sql-statement-show-processlist.md) + - [`SHOW PROCESSLIST`](/sql-statements/sql-statement-show-processlist.md) - [`SHOW PROFILES`](/sql-statements/sql-statement-show-profiles.md) - [`SHOW SCHEMAS`](/sql-statements/sql-statement-show-schemas.md) - [`SHOW STATS_HEALTHY`](/sql-statements/sql-statement-show-stats-healthy.md) @@ -492,6 +587,7 @@ - [Miscellaneous Functions](/functions-and-operators/miscellaneous-functions.md) - [Precision Math](/functions-and-operators/precision-math.md) - [Set Operations](/functions-and-operators/set-operators.md) + - [Sequence Functions](/functions-and-operators/sequence-functions.md) - [List of Expressions for Pushdown](/functions-and-operators/expressions-pushed-down.md) - [TiDB Specific Functions](/functions-and-operators/tidb-functions.md) - [Clustered Indexes](/clustered-indexes.md) @@ -539,6 +635,7 @@ - [`DDL_JOBS`](/information-schema/information-schema-ddl-jobs.md) - [`DEADLOCKS`](/information-schema/information-schema-deadlocks.md) - [`ENGINES`](/information-schema/information-schema-engines.md) + - [`KEYWORDS`](/information-schema/information-schema-keywords.md) - [`KEY_COLUMN_USAGE`](/information-schema/information-schema-key-column-usage.md) - [`MEMORY_USAGE`](/information-schema/information-schema-memory-usage.md) - [`MEMORY_USAGE_OPS_HISTORY`](/information-schema/information-schema-memory-usage-ops-history.md) @@ -576,6 +673,7 @@ - [Metadata Lock](/metadata-lock.md) - [Use UUIDs](/best-practices/uuid.md) - [System Variables](/system-variables.md) + - [Server Status Variables](/status-variables.md) - Storage Engines - TiKV - [TiKV Overview](/tikv-overview.md) @@ -585,18 +683,43 @@ - [Spill to Disk](/tiflash/tiflash-spill-disk.md) - CLI - [Overview](/tidb-cloud/cli-reference.md) - - cluster + - auth + - [login](/tidb-cloud/ticloud-auth-login.md) + - [logout](/tidb-cloud/ticloud-auth-logout.md) + - [whoami](/tidb-cloud/ticloud-auth-whoami.md) + - serverless - [create](/tidb-cloud/ticloud-cluster-create.md) - [delete](/tidb-cloud/ticloud-cluster-delete.md) - [describe](/tidb-cloud/ticloud-cluster-describe.md) - [list](/tidb-cloud/ticloud-cluster-list.md) - - [connect-info](/tidb-cloud/ticloud-cluster-connect-info.md) - - branch - - [create](/tidb-cloud/ticloud-branch-create.md) - - [delete](/tidb-cloud/ticloud-branch-delete.md) - - [describe](/tidb-cloud/ticloud-branch-describe.md) - - [list](/tidb-cloud/ticloud-branch-list.md) - - [connect-info](/tidb-cloud/ticloud-branch-connect-info.md) + - [update](/tidb-cloud/ticloud-serverless-update.md) + - [spending-limit](/tidb-cloud/ticloud-serverless-spending-limit.md) + - [region](/tidb-cloud/ticloud-serverless-region.md) + - [shell](/tidb-cloud/ticloud-serverless-shell.md) + - branch + - [create](/tidb-cloud/ticloud-branch-create.md) + - [delete](/tidb-cloud/ticloud-branch-delete.md) + - [describe](/tidb-cloud/ticloud-branch-describe.md) + - [list](/tidb-cloud/ticloud-branch-list.md) + - [shell](/tidb-cloud/ticloud-branch-shell.md) + - import + - [cancel](/tidb-cloud/ticloud-import-cancel.md) + - [describe](/tidb-cloud/ticloud-import-describe.md) + - [list](/tidb-cloud/ticloud-import-list.md) + - [start](/tidb-cloud/ticloud-import-start.md) + - export + - [create](/tidb-cloud/ticloud-serverless-export-create.md) + - [describe](/tidb-cloud/ticloud-serverless-export-describe.md) + - [list](/tidb-cloud/ticloud-serverless-export-list.md) + - [cancel](/tidb-cloud/ticloud-serverless-export-cancel.md) + - [download](/tidb-cloud/ticloud-serverless-export-download.md) + - sql-user + - [create](/tidb-cloud/ticloud-serverless-sql-user-create.md) + - [delete](/tidb-cloud/ticloud-serverless-sql-user-delete.md) + - [list](/tidb-cloud/ticloud-serverless-sql-user-list.md) + - [update](/tidb-cloud/ticloud-serverless-sql-user-update.md) + - [ai](/tidb-cloud/ticloud-ai.md) + - [completion](/tidb-cloud/ticloud-completion.md) - config - [create](/tidb-cloud/ticloud-config-create.md) - [delete](/tidb-cloud/ticloud-config-delete.md) @@ -605,33 +728,32 @@ - [list](/tidb-cloud/ticloud-config-list.md) - [set](/tidb-cloud/ticloud-config-set.md) - [use](/tidb-cloud/ticloud-config-use.md) - - [connect](/tidb-cloud/ticloud-connect.md) - - import - - [cancel](/tidb-cloud/ticloud-import-cancel.md) - - [describe](/tidb-cloud/ticloud-import-describe.md) - - [list](/tidb-cloud/ticloud-import-list.md) - - start - - [local](/tidb-cloud/ticloud-import-start-local.md) - - [s3](/tidb-cloud/ticloud-import-start-s3.md) - - [mysql](/tidb-cloud/ticloud-import-start-mysql.md) - project - [list](/tidb-cloud/ticloud-project-list.md) - - [update](/tidb-cloud/ticloud-update.md) + - [upgrade](/tidb-cloud/ticloud-upgrade.md) + - [help](/tidb-cloud/ticloud-help.md) - [Table Filter](/table-filter.md) - [Resource Control](/tidb-resource-control.md) - - [TiDB Backend Task Distributed Execution Framework](/tidb-distributed-execution-framework.md) - - [TiDB Global Sort](/tidb-global-sort.md) - - [DDL Execution Principles and Best Practices](/ddl-introduction.md) + - [URI Formats of External Storage Services](/external-storage-uri.md) + - [DDL Execution Principles and Best Practices](/best-practices/ddl-introduction.md) - [Troubleshoot Inconsistency Between Data and Indexes](/troubleshoot-data-inconsistency-errors.md) - [Support](/tidb-cloud/tidb-cloud-support.md) - [Glossary](/tidb-cloud/tidb-cloud-glossary.md) - FAQs - [TiDB Cloud FAQs](/tidb-cloud/tidb-cloud-faq.md) - - [TiDB Serverless FAQs](/tidb-cloud/serverless-faqs.md) + - [TiDB Cloud Serverless FAQs](/tidb-cloud/serverless-faqs.md) - Release Notes - - [2023](/tidb-cloud/tidb-cloud-release-notes.md) + - [2024](/tidb-cloud/tidb-cloud-release-notes.md) + - [2023](/tidb-cloud/release-notes-2023.md) - [2022](/tidb-cloud/release-notes-2022.md) - [2021](/tidb-cloud/release-notes-2021.md) - [2020](/tidb-cloud/release-notes-2020.md) - Maintenance Notification + - [[2024-09-15] TiDB Cloud Console Maintenance Notification](/tidb-cloud/notification-2024-09-15-console-maintenance.md) + - [[2024-04-18] TiDB Cloud Data Migration (DM) Feature Maintenance Notification](/tidb-cloud/notification-2024-04-18-dm-feature-maintenance.md) + - [[2024-04-16] TiDB Cloud Monitoring Features Maintenance Notification](/tidb-cloud/notification-2024-04-16-monitoring-features-maintenance.md) + - [[2024-04-11] TiDB Cloud Data Migration (DM) Feature Maintenance Notification](/tidb-cloud/notification-2024-04-11-dm-feature-maintenance.md) + - [[2024-04-09] TiDB Cloud Monitoring Features Maintenance Notification](/tidb-cloud/notification-2024-04-09-monitoring-features-maintenance.md) + - [[2023-11-14] TiDB Cloud Dedicated Scale Feature Maintenance Notification](/tidb-cloud/notification-2023-11-14-scale-feature-maintenance.md) + - [[2023-09-26] TiDB Cloud Console Maintenance Notification](/tidb-cloud/notification-2023-09-26-console-maintenance.md) - [[2023-08-31] TiDB Cloud Console Maintenance Notification](/tidb-cloud/notification-2023-08-31-console-maintenance.md) diff --git a/TOC.md b/TOC.md index f83b17bb98b91..ae05dad377020 100644 --- a/TOC.md +++ b/TOC.md @@ -1,113 +1,19 @@ -- [Docs Home](https://docs.pingcap.com/) -- About TiDB - - [TiDB Introduction](/overview.md) - - [TiDB 7.4 Release Notes](/releases/release-7.4.0.md) +- About TiDB Self-Managed + - [What is TiDB Self-Managed](/overview.md) + - [TiDB 7.5 Release Notes](/releases/release-7.5.0.md) - [Features](/basic-features.md) - [MySQL Compatibility](/mysql-compatibility.md) - [TiDB Limitations](/tidb-limitations.md) - [Credits](/credits.md) - - [Roadmap](/tidb-roadmap.md) - Quick Start - [Try Out TiDB](/quick-start-with-tidb.md) - [Try Out HTAP](/quick-start-with-htap.md) - [Learn TiDB SQL](/basic-sql-operations.md) - [Learn HTAP](/explore-htap.md) - [Import Example Database](/import-example-data.md) -- Develop - - [Overview](/develop/dev-guide-overview.md) - - Quick Start - - [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md) - - [CRUD SQL in TiDB](/develop/dev-guide-tidb-crud-sql.md) - - Example Applications - - Java - - [JDBC](/develop/dev-guide-sample-application-java-jdbc.md) - - [MyBatis](/develop/dev-guide-sample-application-java-mybatis.md) - - [Hibernate](/develop/dev-guide-sample-application-java-hibernate.md) - - [Spring Boot](/develop/dev-guide-sample-application-java-spring-boot.md) - - Go - - [Go-MySQL-Driver](/develop/dev-guide-sample-application-golang-sql-driver.md) - - [GORM](/develop/dev-guide-sample-application-golang-gorm.md) - - Python - - [mysqlclient](/develop/dev-guide-sample-application-python-mysqlclient.md) - - [MySQL Connector/Python](/develop/dev-guide-sample-application-python-mysql-connector.md) - - [PyMySQL](/develop/dev-guide-sample-application-python-pymysql.md) - - [SQLAlchemy](/develop/dev-guide-sample-application-python-sqlalchemy.md) - - [peewee](/develop/dev-guide-sample-application-python-peewee.md) - - [Django](/develop/dev-guide-sample-application-python-django.md) - - Node.js - - [node-mysql2](/develop/dev-guide-sample-application-nodejs-mysql2.md) - - [mysql.js](/develop/dev-guide-sample-application-nodejs-mysqljs.md) - - [Prisma](/develop/dev-guide-sample-application-nodejs-prisma.md) - - [Sequelize](/develop/dev-guide-sample-application-nodejs-sequelize.md) - - [TypeORM](/develop/dev-guide-sample-application-nodejs-typeorm.md) - - [Next.js](/develop/dev-guide-sample-application-nextjs.md) - - [AWS Lambda](/develop/dev-guide-sample-application-aws-lambda.md) - - Ruby - - [mysql2](/develop/dev-guide-sample-application-ruby-mysql2.md) - - [Rails](/develop/dev-guide-sample-application-ruby-rails.md) - - Connect to TiDB - - GUI Database Tools - - [JetBrains DataGrip](/develop/dev-guide-gui-datagrip.md) - - [DBeaver](/develop/dev-guide-gui-dbeaver.md) - - [VS Code](/develop/dev-guide-gui-vscode-sqltools.md) - - [Choose Driver or ORM](/develop/dev-guide-choose-driver-or-orm.md) - - [Connect to TiDB](/develop/dev-guide-connect-to-tidb.md) - - [Connection Pools and Connection Parameters](/develop/dev-guide-connection-parameters.md) - - Design Database Schema - - [Overview](/develop/dev-guide-schema-design-overview.md) - - [Create a Database](/develop/dev-guide-create-database.md) - - [Create a Table](/develop/dev-guide-create-table.md) - - [Create a Secondary Index](/develop/dev-guide-create-secondary-indexes.md) - - Write Data - - [Insert Data](/develop/dev-guide-insert-data.md) - - [Update Data](/develop/dev-guide-update-data.md) - - [Delete Data](/develop/dev-guide-delete-data.md) - - [Periodically Delete Data Using Time to Live](/time-to-live.md) - - [Prepared Statements](/develop/dev-guide-prepared-statement.md) - - Read Data - - [Query Data from a Single Table](/develop/dev-guide-get-data-from-single-table.md) - - [Multi-table Join Queries](/develop/dev-guide-join-tables.md) - - [Subquery](/develop/dev-guide-use-subqueries.md) - - [Paginate Results](/develop/dev-guide-paginate-results.md) - - [Views](/develop/dev-guide-use-views.md) - - [Temporary Tables](/develop/dev-guide-use-temporary-tables.md) - - [Common Table Expression](/develop/dev-guide-use-common-table-expression.md) - - Read Replica Data - - [Follower Read](/develop/dev-guide-use-follower-read.md) - - [Stale Read](/develop/dev-guide-use-stale-read.md) - - [HTAP Queries](/develop/dev-guide-hybrid-oltp-and-olap-queries.md) - - Transaction - - [Overview](/develop/dev-guide-transaction-overview.md) - - [Optimistic and Pessimistic Transactions](/develop/dev-guide-optimistic-and-pessimistic-transaction.md) - - [Transaction Restraints](/develop/dev-guide-transaction-restraints.md) - - [Handle Transaction Errors](/develop/dev-guide-transaction-troubleshoot.md) - - Optimize - - [Overview](/develop/dev-guide-optimize-sql-overview.md) - - [SQL Performance Tuning](/develop/dev-guide-optimize-sql.md) - - [Best Practices for Performance Tuning](/develop/dev-guide-optimize-sql-best-practices.md) - - [Best Practices for Indexing](/develop/dev-guide-index-best-practice.md) - - Other Optimization Methods - - [Avoid Implicit Type Conversions](/develop/dev-guide-implicit-type-conversion.md) - - [Unique Serial Number Generation](/develop/dev-guide-unique-serial-number-generation.md) - - Troubleshoot - - [SQL or Transaction Issues](/develop/dev-guide-troubleshoot-overview.md) - - [Unstable Result Set](/develop/dev-guide-unstable-result-set.md) - - [Timeouts](/develop/dev-guide-timeouts-in-tidb.md) - - Reference - - [Bookshop Example Application](/develop/dev-guide-bookshop-schema-design.md) - - Guidelines - - [Object Naming Convention](/develop/dev-guide-object-naming-guidelines.md) - - [SQL Development Specifications](/develop/dev-guide-sql-development-specification.md) - - Cloud Native Development Environment - - [Gitpod](/develop/dev-guide-playground-gitpod.md) - - Third-Party Support - - [Third-Party Tools Supported by TiDB](/develop/dev-guide-third-party-support.md) - - [Known Incompatibility Issues with Third-Party Tools](/develop/dev-guide-third-party-tools-compatibility.md) - - [ProxySQL Integration Guide](/develop/dev-guide-proxysql-integration.md) - - [Amazon AppFlow Integration Guide](/develop/dev-guide-aws-appflow-integration.md) - Deploy - [Software and Hardware Requirements](/hardware-and-software-requirements.md) - [Environment Configuration Checklist](/check-before-deployment.md) @@ -133,10 +39,11 @@ - [Import Best Practices](/tidb-lightning/data-import-best-practices.md) - Migration Scenarios - [Migrate from Aurora](/migrate-aurora-to-tidb.md) - - [Migrate MySQL of Small Datasets](/migrate-small-mysql-to-tidb.md) - - [Migrate MySQL of Large Datasets](/migrate-large-mysql-to-tidb.md) + - [Migrate Small Datasets from MySQL](/migrate-small-mysql-to-tidb.md) + - [Migrate Large Datasets from MySQL](/migrate-large-mysql-to-tidb.md) - [Migrate and Merge MySQL Shards of Small Datasets](/migrate-small-mysql-shards-to-tidb.md) - [Migrate and Merge MySQL Shards of Large Datasets](/migrate-large-mysql-shards-to-tidb.md) + - [Migrate from MariaDB](/migrate-from-mariadb.md) - [Migrate from CSV Files](/migrate-from-csv-files-to-tidb.md) - [Migrate from SQL Files](/migrate-from-sql-files-to-tidb.md) - [Migrate from Parquet Files](/migrate-from-parquet-files-to-tidb.md) @@ -153,10 +60,19 @@ - [Integrate with Confluent and Snowflake](/ticdc/integrate-confluent-using-ticdc.md) - [Integrate with Apache Kafka and Apache Flink](/replicate-data-to-kafka.md) - Maintain + - Security + - [Best Practices for TiDB Security Configuration](/best-practices-for-security-configuration.md) + - [Enable TLS Between TiDB Clients and Servers](/enable-tls-between-clients-and-servers.md) + - [Enable TLS Between TiDB Components](/enable-tls-between-components.md) + - [Generate Self-signed Certificates](/generate-self-signed-certificates.md) + - [Encryption at Rest](/encryption-at-rest.md) + - [Enable Encryption for Disk Spill](/enable-disk-spill-encrypt.md) + - [Log Redaction](/log-redaction.md) - Upgrade - [Use TiUP](/upgrade-tidb-using-tiup.md) - [Use TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/stable/upgrade-a-tidb-cluster) - [TiDB Smooth Upgrade](/smooth-upgrade-tidb.md) + - [Migrate and Upgrade a TiDB Cluster](/tidb-upgrade-migration-guide.md) - [TiFlash Upgrade Guide](/tiflash-upgrade-guide.md) - Scale - [Use TiUP (Recommended)](/scale-tidb-using-tiup.md) @@ -204,6 +120,37 @@ - [Monitoring API](/tidb-monitoring-api.md) - [Deploy Monitoring Services](/deploy-monitoring-services.md) - [Upgrade Monitoring Services](/upgrade-monitoring-services.md) + - TiDB Dashboard + - [Overview](/dashboard/dashboard-intro.md) + - Maintain + - [Deploy](/dashboard/dashboard-ops-deploy.md) + - [Reverse Proxy](/dashboard/dashboard-ops-reverse-proxy.md) + - [User Management](/dashboard/dashboard-user.md) + - [Secure](/dashboard/dashboard-ops-security.md) + - [Access](/dashboard/dashboard-access.md) + - [Overview Page](/dashboard/dashboard-overview.md) + - [Cluster Info Page](/dashboard/dashboard-cluster-info.md) + - [Top SQL Page](/dashboard/top-sql.md) + - [Key Visualizer Page](/dashboard/dashboard-key-visualizer.md) + - [Metrics Relation Graph](/dashboard/dashboard-metrics-relation.md) + - SQL Statements Analysis + - [SQL Statements Page](/dashboard/dashboard-statement-list.md) + - [SQL Details Page](/dashboard/dashboard-statement-details.md) + - [Slow Queries Page](/dashboard/dashboard-slow-query.md) + - Cluster Diagnostics + - [Access Cluster Diagnostics Page](/dashboard/dashboard-diagnostics-access.md) + - [View Diagnostics Report](/dashboard/dashboard-diagnostics-report.md) + - [Use Diagnostics](/dashboard/dashboard-diagnostics-usage.md) + - [Monitoring Page](/dashboard/dashboard-monitoring.md) + - [Search Logs Page](/dashboard/dashboard-log-search.md) + - [Resource Manager Page](/dashboard/dashboard-resource-manager.md) + - Instance Profiling + - [Manual Profiling](/dashboard/dashboard-profiling.md) + - [Continuous Profiling](/dashboard/continuous-profiling.md) + - Session Management and Configuration + - [Share Session](/dashboard/dashboard-session-share.md) + - [Configure SSO](/dashboard/dashboard-session-sso.md) + - [FAQ](/dashboard/dashboard-faq.md) - [Export Grafana Snapshots](/exporting-grafana-snapshots.md) - [TiDB Cluster Alert Rules](/alert-rules.md) - [TiFlash Alert Rules](/tiflash/tiflash-alert-rules.md) @@ -241,7 +188,6 @@ - [TiFlash Performance Analysis Methods](/tiflash-performance-tuning-methods.md) - [TiCDC Performance Analysis Methods](/ticdc-performance-tuning-methods.md) - [Latency Breakdown](/latency-breakdown.md) - - [TiDB Best Practices on Public Cloud](/best-practices-on-public-cloud.md) - Configuration Tuning - [Tune Operating System Performance](/tune-operating-system.md) - [Tune TiDB Memory](/configure-memory-usage.md) @@ -308,22 +254,9 @@ - [Perform Stale Read Using `tidb_read_staleness`](/tidb-read-staleness.md) - [Perform Stale Read Using `tidb_external_ts`](/tidb-external-ts.md) - [Use the `tidb_snapshot` System Variable](/read-historical-data.md) - - Best Practices - - [Use TiDB](/best-practices/tidb-best-practices.md) - - [Java Application Development](/best-practices/java-app-best-practices.md) - - [Use HAProxy](/best-practices/haproxy-best-practices.md) - - [Highly Concurrent Write](/best-practices/high-concurrency-best-practices.md) - - [Grafana Monitoring](/best-practices/grafana-monitor-best-practices.md) - - [PD Scheduling](/best-practices/pd-scheduling-best-practices.md) - - [TiKV Performance Tuning with Massive Regions](/best-practices/massive-regions-best-practices.md) - - [Three-node Hybrid Deployment](/best-practices/three-nodes-hybrid-deployment.md) - - [Local Read Under Three Data Centers Deployment](/best-practices/three-dc-local-read.md) - - [Use UUIDs](/best-practices/uuid.md) - - [Read-Only Storage Nodes](/best-practices/readonly-nodes.md) - [Use Placement Rules](/configure-placement-rules.md) - [Use Load Base Split](/configure-load-base-split.md) - [Use Store Limit](/configure-store-limit.md) - - [DDL Execution Principles and Best Practices](/ddl-introduction.md) - TiDB Tools - [Overview](/ecosystem-tool-user-guide.md) - [Use Cases](/ecosystem-tool-user-case.md) @@ -388,6 +321,7 @@ - [tiup cluster start](/tiup/tiup-component-cluster-start.md) - [tiup cluster stop](/tiup/tiup-component-cluster-stop.md) - [tiup cluster template](/tiup/tiup-component-cluster-template.md) + - [tiup cluster tls](/tiup/tiup-component-cluster-tls.md) - [tiup cluster upgrade](/tiup/tiup-component-cluster-upgrade.md) - TiUP DM Commands - [Overview](/tiup/tiup-component-dm.md) @@ -431,7 +365,7 @@ - [Use TiUP (Recommended)](/dm/deploy-a-dm-cluster-using-tiup.md) - [Use TiUP Offline](/dm/deploy-a-dm-cluster-using-tiup-offline.md) - [Use Binary](/dm/deploy-a-dm-cluster-using-binary.md) - - [Use Kubernetes](https://docs.pingcap.com/tidb-in-kubernetes/dev/deploy-tidb-dm) + - [Use Kubernetes](https://docs.pingcap.com/tidb-in-kubernetes/v1.5/deploy-tidb-dm) - Tutorials - [Create a Data Source](/dm/quick-start-create-source.md) - [Manage Data Sources](/dm/dm-manage-source.md) @@ -520,10 +454,13 @@ - [Target Database Requirements](/tidb-lightning/tidb-lightning-requirements.md) - Data Sources - [Data Match Rules](/tidb-lightning/tidb-lightning-data-source.md) + - [Rename databases and tables](/tidb-lightning/tidb-lightning-data-source.md#rename-databases-and-tables) - [CSV](/tidb-lightning/tidb-lightning-data-source.md#csv) - [SQL](/tidb-lightning/tidb-lightning-data-source.md#sql) - [Parquet](/tidb-lightning/tidb-lightning-data-source.md#parquet) + - [Compressed files](/tidb-lightning/tidb-lightning-data-source.md#compressed-files) - [Customized File](/tidb-lightning/tidb-lightning-data-source.md#match-customized-files) + - [Import data from Amazon S3](/tidb-lightning/tidb-lightning-data-source.md#import-data-from-amazon-s3) - Physical Import Mode - [Requirements and Limitations](/tidb-lightning/tidb-lightning-physical-import-mode.md) - [Use Physical Import Mode](/tidb-lightning/tidb-lightning-physical-import-mode-usage.md) @@ -560,12 +497,14 @@ - [Bidirectional Replication](/ticdc/ticdc-bidirectional-replication.md) - [Data Integrity Validation for Single-Row Data](/ticdc/ticdc-integrity-check.md) - [Data Consistency Validation for TiDB Upstream/Downstream Clusters](/ticdc/ticdc-upstream-downstream-check.md) + - [TiCDC Behavior in Splitting UPDATE Events](/ticdc/ticdc-split-update-behavior.md) - Monitor and Alert - [Monitoring Metrics Summary](/ticdc/ticdc-summary-monitor.md) - [Monitoring Metrics Details](/ticdc/monitor-ticdc.md) - [Alert Rules](/ticdc/ticdc-alert-rules.md) - Reference - [Architecture](/ticdc/ticdc-architecture.md) + - [TiCDC Data Replication Capabilities](/ticdc/ticdc-data-replication-capabilities.md) - [TiCDC Server Configurations](/ticdc/ticdc-server-config.md) - [TiCDC Changefeed Configurations](/ticdc/ticdc-changefeed-config.md) - Output Protocols @@ -640,9 +579,14 @@ - [TiFlash Late Materialization](/tiflash/tiflash-late-materialization.md) - [Spill to Disk](/tiflash/tiflash-spill-disk.md) - [Data Validation](/tiflash/tiflash-data-validation.md) + - [MinTSO Scheduler](/tiflash/tiflash-mintso-scheduler.md) - [Compatibility](/tiflash/tiflash-compatibility.md) - [Pipeline Execution Model](/tiflash/tiflash-pipeline-model.md) + - TiDB Distributed eXecution Framework (DXF) + - [Introduction](/tidb-distributed-execution-framework.md) + - [TiDB Global Sort](/tidb-global-sort.md) - [System Variables](/system-variables.md) + - [Server Status Variables](/status-variables.md) - Configuration File Parameters - [tidb-server](/tidb-configuration-file.md) - [tikv-server](/tikv-configuration-file.md) @@ -667,13 +611,6 @@ - [TiFlash](/tiflash/monitor-tiflash.md) - [TiCDC](/ticdc/monitor-ticdc.md) - [Resource Control](/grafana-resource-control-dashboard.md) - - Security - - [Enable TLS Between TiDB Clients and Servers](/enable-tls-between-clients-and-servers.md) - - [Enable TLS Between TiDB Components](/enable-tls-between-components.md) - - [Generate Self-signed Certificates](/generate-self-signed-certificates.md) - - [Encryption at Rest](/encryption-at-rest.md) - - [Enable Encryption for Disk Spill](/enable-disk-spill-encrypt.md) - - [Log Redaction](/log-redaction.md) - Privileges - [Security Compatibility with MySQL](/security-compatibility-with-mysql.md) - [Privilege Management](/privilege-management.md) @@ -686,6 +623,7 @@ - Attributes - [AUTO_INCREMENT](/auto-increment.md) - [AUTO_RANDOM](/auto-random.md) + - [_tidb_rowid](/tidb-rowid.md) - [SHARD_ROW_ID_BITS](/shard-row-id-bits.md) - [Literal Values](/literal-values.md) - [Schema Object Names](/schema-object-names.md) @@ -694,8 +632,6 @@ - [Expression Syntax](/expression-syntax.md) - [Comment Syntax](/comment-syntax.md) - SQL Statements - - [`ADD COLUMN`](/sql-statements/sql-statement-add-column.md) - - [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) - [`ADMIN`](/sql-statements/sql-statement-admin.md) - [`ADMIN CANCEL DDL`](/sql-statements/sql-statement-admin-cancel-ddl.md) - [`ADMIN CHECKSUM TABLE`](/sql-statements/sql-statement-admin-checksum-table.md) @@ -707,13 +643,22 @@ - [`ADMIN SHOW DDL [JOBS|JOB QUERIES]`](/sql-statements/sql-statement-admin-show-ddl.md) - [`ADMIN SHOW TELEMETRY`](/sql-statements/sql-statement-admin-show-telemetry.md) - [`ALTER DATABASE`](/sql-statements/sql-statement-alter-database.md) - - [`ALTER INDEX`](/sql-statements/sql-statement-alter-index.md) - [`ALTER INSTANCE`](/sql-statements/sql-statement-alter-instance.md) - [`ALTER PLACEMENT POLICY`](/sql-statements/sql-statement-alter-placement-policy.md) - [`ALTER RANGE`](/sql-statements/sql-statement-alter-range.md) - [`ALTER RESOURCE GROUP`](/sql-statements/sql-statement-alter-resource-group.md) - - [`ALTER TABLE`](/sql-statements/sql-statement-alter-table.md) - - [`ALTER TABLE COMPACT`](/sql-statements/sql-statement-alter-table-compact.md) + - [`ALTER SEQUENCE`](/sql-statements/sql-statement-alter-sequence.md) + - `ALTER TABLE` + - [Overview](/sql-statements/sql-statement-alter-table.md) + - [`ADD COLUMN`](/sql-statements/sql-statement-add-column.md) + - [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) + - [`ALTER INDEX`](/sql-statements/sql-statement-alter-index.md) + - [`CHANGE COLUMN`](/sql-statements/sql-statement-change-column.md) + - [`COMPACT`](/sql-statements/sql-statement-alter-table-compact.md) + - [`DROP COLUMN`](/sql-statements/sql-statement-drop-column.md) + - [`DROP INDEX`](/sql-statements/sql-statement-drop-index.md) + - [`MODIFY COLUMN`](/sql-statements/sql-statement-modify-column.md) + - [`RENAME INDEX`](/sql-statements/sql-statement-rename-index.md) - [`ALTER USER`](/sql-statements/sql-statement-alter-user.md) - [`ANALYZE TABLE`](/sql-statements/sql-statement-analyze-table.md) - [`BACKUP`](/sql-statements/sql-statement-backup.md) @@ -721,11 +666,10 @@ - [`BEGIN`](/sql-statements/sql-statement-begin.md) - [`CALIBRATE RESOURCE`](/sql-statements/sql-statement-calibrate-resource.md) - [`CANCEL IMPORT JOB`](/sql-statements/sql-statement-cancel-import-job.md) - - [`CHANGE COLUMN`](/sql-statements/sql-statement-change-column.md) - [`COMMIT`](/sql-statements/sql-statement-commit.md) - [`CHANGE DRAINER`](/sql-statements/sql-statement-change-drainer.md) - [`CHANGE PUMP`](/sql-statements/sql-statement-change-pump.md) - - [`CREATE [GLOBAL|SESSION] BINDING`](/sql-statements/sql-statement-create-binding.md) + - [`CREATE BINDING`](/sql-statements/sql-statement-create-binding.md) - [`CREATE DATABASE`](/sql-statements/sql-statement-create-database.md) - [`CREATE INDEX`](/sql-statements/sql-statement-create-index.md) - [`CREATE PLACEMENT POLICY`](/sql-statements/sql-statement-create-placement-policy.md) @@ -741,10 +685,8 @@ - [`DESC`](/sql-statements/sql-statement-desc.md) - [`DESCRIBE`](/sql-statements/sql-statement-describe.md) - [`DO`](/sql-statements/sql-statement-do.md) - - [`DROP [GLOBAL|SESSION] BINDING`](/sql-statements/sql-statement-drop-binding.md) - - [`DROP COLUMN`](/sql-statements/sql-statement-drop-column.md) + - [`DROP BINDING`](/sql-statements/sql-statement-drop-binding.md) - [`DROP DATABASE`](/sql-statements/sql-statement-drop-database.md) - - [`DROP INDEX`](/sql-statements/sql-statement-drop-index.md) - [`DROP PLACEMENT POLICY`](/sql-statements/sql-statement-drop-placement-policy.md) - [`DROP RESOURCE GROUP`](/sql-statements/sql-statement-drop-resource-group.md) - [`DROP ROLE`](/sql-statements/sql-statement-drop-role.md) @@ -756,7 +698,7 @@ - [`EXECUTE`](/sql-statements/sql-statement-execute.md) - [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) - [`EXPLAIN`](/sql-statements/sql-statement-explain.md) - - [`FLASHBACK CLUSTER TO TIMESTAMP`](/sql-statements/sql-statement-flashback-to-timestamp.md) + - [`FLASHBACK CLUSTER`](/sql-statements/sql-statement-flashback-cluster.md) - [`FLASHBACK DATABASE`](/sql-statements/sql-statement-flashback-database.md) - [`FLASHBACK TABLE`](/sql-statements/sql-statement-flashback-table.md) - [`FLUSH PRIVILEGES`](/sql-statements/sql-statement-flush-privileges.md) @@ -766,17 +708,15 @@ - [`GRANT `](/sql-statements/sql-statement-grant-role.md) - [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) - [`INSERT`](/sql-statements/sql-statement-insert.md) - - [`KILL [TIDB]`](/sql-statements/sql-statement-kill.md) + - [`KILL`](/sql-statements/sql-statement-kill.md) - [`LOAD DATA`](/sql-statements/sql-statement-load-data.md) - [`LOAD STATS`](/sql-statements/sql-statement-load-stats.md) - [`LOCK STATS`](/sql-statements/sql-statement-lock-stats.md) - - [`LOCK TABLES` and `UNLOCK TABLES`](/sql-statements/sql-statement-lock-tables-and-unlock-tables.md) - - [`MODIFY COLUMN`](/sql-statements/sql-statement-modify-column.md) + - [`[LOCK|UNLOCK] TABLES`](/sql-statements/sql-statement-lock-tables-and-unlock-tables.md) - [`PREPARE`](/sql-statements/sql-statement-prepare.md) - [`QUERY WATCH`](/sql-statements/sql-statement-query-watch.md) - [`RECOVER TABLE`](/sql-statements/sql-statement-recover-table.md) - [`RENAME USER`](/sql-statements/sql-statement-rename-user.md) - - [`RENAME INDEX`](/sql-statements/sql-statement-rename-index.md) - [`RENAME TABLE`](/sql-statements/sql-statement-rename-table.md) - [`REPLACE`](/sql-statements/sql-statement-replace.md) - [`RESTORE`](/sql-statements/sql-statement-restore.md) @@ -791,14 +731,14 @@ - [`SET RESOURCE GROUP`](/sql-statements/sql-statement-set-resource-group.md) - [`SET ROLE`](/sql-statements/sql-statement-set-role.md) - [`SET TRANSACTION`](/sql-statements/sql-statement-set-transaction.md) - - [`SET [GLOBAL|SESSION] `](/sql-statements/sql-statement-set-variable.md) + - [`SET `](/sql-statements/sql-statement-set-variable.md) - [`SHOW ANALYZE STATUS`](/sql-statements/sql-statement-show-analyze-status.md) - [`SHOW [BACKUPS|RESTORES]`](/sql-statements/sql-statement-show-backups.md) - - [`SHOW [GLOBAL|SESSION] BINDINGS`](/sql-statements/sql-statement-show-bindings.md) + - [`SHOW BINDINGS`](/sql-statements/sql-statement-show-bindings.md) - [`SHOW BUILTINS`](/sql-statements/sql-statement-show-builtins.md) - [`SHOW CHARACTER SET`](/sql-statements/sql-statement-show-character-set.md) - [`SHOW COLLATION`](/sql-statements/sql-statement-show-collation.md) - - [`SHOW [FULL] COLUMNS FROM`](/sql-statements/sql-statement-show-columns-from.md) + - [`SHOW COLUMNS FROM`](/sql-statements/sql-statement-show-columns-from.md) - [`SHOW CONFIG`](/sql-statements/sql-statement-show-config.md) - [`SHOW CREATE DATABASE`](/sql-statements/sql-statement-show-create-database.md) - [`SHOW CREATE PLACEMENT POLICY`](/sql-statements/sql-statement-show-create-placement-policy.md) @@ -810,19 +750,17 @@ - [`SHOW DRAINER STATUS`](/sql-statements/sql-statement-show-drainer-status.md) - [`SHOW ENGINES`](/sql-statements/sql-statement-show-engines.md) - [`SHOW ERRORS`](/sql-statements/sql-statement-show-errors.md) - - [`SHOW [FULL] FIELDS FROM`](/sql-statements/sql-statement-show-fields-from.md) + - [`SHOW FIELDS FROM`](/sql-statements/sql-statement-show-fields-from.md) - [`SHOW GRANTS`](/sql-statements/sql-statement-show-grants.md) - [`SHOW IMPORT JOB`](/sql-statements/sql-statement-show-import-job.md) - - [`SHOW INDEX [FROM|IN]`](/sql-statements/sql-statement-show-index.md) - - [`SHOW INDEXES [FROM|IN]`](/sql-statements/sql-statement-show-indexes.md) - - [`SHOW KEYS [FROM|IN]`](/sql-statements/sql-statement-show-keys.md) + - [`SHOW INDEXES`](/sql-statements/sql-statement-show-indexes.md) - [`SHOW MASTER STATUS`](/sql-statements/sql-statement-show-master-status.md) - [`SHOW PLACEMENT`](/sql-statements/sql-statement-show-placement.md) - [`SHOW PLACEMENT FOR`](/sql-statements/sql-statement-show-placement-for.md) - [`SHOW PLACEMENT LABELS`](/sql-statements/sql-statement-show-placement-labels.md) - [`SHOW PLUGINS`](/sql-statements/sql-statement-show-plugins.md) - [`SHOW PRIVILEGES`](/sql-statements/sql-statement-show-privileges.md) - - [`SHOW [FULL] PROCESSSLIST`](/sql-statements/sql-statement-show-processlist.md) + - [`SHOW PROCESSLIST`](/sql-statements/sql-statement-show-processlist.md) - [`SHOW PROFILES`](/sql-statements/sql-statement-show-profiles.md) - [`SHOW PUMP STATUS`](/sql-statements/sql-statement-show-pump-status.md) - [`SHOW SCHEMAS`](/sql-statements/sql-statement-show-schemas.md) @@ -834,8 +772,8 @@ - [`SHOW TABLE NEXT_ROW_ID`](/sql-statements/sql-statement-show-table-next-rowid.md) - [`SHOW TABLE REGIONS`](/sql-statements/sql-statement-show-table-regions.md) - [`SHOW TABLE STATUS`](/sql-statements/sql-statement-show-table-status.md) - - [`SHOW [FULL] TABLES`](/sql-statements/sql-statement-show-tables.md) - - [`SHOW [GLOBAL|SESSION] VARIABLES`](/sql-statements/sql-statement-show-variables.md) + - [`SHOW TABLES`](/sql-statements/sql-statement-show-tables.md) + - [`SHOW VARIABLES`](/sql-statements/sql-statement-show-variables.md) - [`SHOW WARNINGS`](/sql-statements/sql-statement-show-warnings.md) - [`SHUTDOWN`](/sql-statements/sql-statement-shutdown.md) - [`SPLIT REGION`](/sql-statements/sql-statement-split-region.md) @@ -874,6 +812,7 @@ - [Miscellaneous Functions](/functions-and-operators/miscellaneous-functions.md) - [Precision Math](/functions-and-operators/precision-math.md) - [Set Operations](/functions-and-operators/set-operators.md) + - [Sequence Functions](/functions-and-operators/sequence-functions.md) - [List of Expressions for Pushdown](/functions-and-operators/expressions-pushed-down.md) - [TiDB Specific Functions](/functions-and-operators/tidb-functions.md) - [Comparisons between Functions and Syntax of Oracle and TiDB](/oracle-functions-to-tidb.md) @@ -923,6 +862,7 @@ - [`INSPECTION_RESULT`](/information-schema/information-schema-inspection-result.md) - [`INSPECTION_RULES`](/information-schema/information-schema-inspection-rules.md) - [`INSPECTION_SUMMARY`](/information-schema/information-schema-inspection-summary.md) + - [`KEYWORDS`](/information-schema/information-schema-keywords.md) - [`KEY_COLUMN_USAGE`](/information-schema/information-schema-key-column-usage.md) - [`MEMORY_USAGE`](/information-schema/information-schema-memory-usage.md) - [`MEMORY_USAGE_OPS_HISTORY`](/information-schema/information-schema-memory-usage-ops-history.md) @@ -962,46 +902,12 @@ - [Overview](/performance-schema/performance-schema.md) - [`SESSION_CONNECT_ATTRS`](/performance-schema/performance-schema-session-connect-attrs.md) - [Metadata Lock](/metadata-lock.md) - - UI - - TiDB Dashboard - - [Overview](/dashboard/dashboard-intro.md) - - Maintain - - [Deploy](/dashboard/dashboard-ops-deploy.md) - - [Reverse Proxy](/dashboard/dashboard-ops-reverse-proxy.md) - - [User Management](/dashboard/dashboard-user.md) - - [Secure](/dashboard/dashboard-ops-security.md) - - [Access](/dashboard/dashboard-access.md) - - [Overview Page](/dashboard/dashboard-overview.md) - - [Cluster Info Page](/dashboard/dashboard-cluster-info.md) - - [Top SQL Page](/dashboard/top-sql.md) - - [Key Visualizer Page](/dashboard/dashboard-key-visualizer.md) - - [Metrics Relation Graph](/dashboard/dashboard-metrics-relation.md) - - SQL Statements Analysis - - [SQL Statements Page](/dashboard/dashboard-statement-list.md) - - [SQL Details Page](/dashboard/dashboard-statement-details.md) - - [Slow Queries Page](/dashboard/dashboard-slow-query.md) - - Cluster Diagnostics - - [Access Cluster Diagnostics Page](/dashboard/dashboard-diagnostics-access.md) - - [View Diagnostics Report](/dashboard/dashboard-diagnostics-report.md) - - [Use Diagnostics](/dashboard/dashboard-diagnostics-usage.md) - - [Monitoring Page](/dashboard/dashboard-monitoring.md) - - [Search Logs Page](/dashboard/dashboard-log-search.md) - - [Resource Manager Page](/dashboard/dashboard-resource-manager.md) - - Instance Profiling - - [Manual Profiling](/dashboard/dashboard-profiling.md) - - [Continuous Profiling](/dashboard/continuous-profiling.md) - - Session Management and Configuration - - [Share Session](/dashboard/dashboard-session-share.md) - - [Configure SSO](/dashboard/dashboard-session-sso.md) - - [FAQ](/dashboard/dashboard-faq.md) - [Telemetry](/telemetry.md) - [Error Codes](/error-codes.md) + - [TiDB Installation Packages](/binary-package.md) - [Table Filter](/table-filter.md) - [Schedule Replicas by Topology Labels](/schedule-replicas-by-topology-labels.md) - [URI Formats of External Storage Services](/external-storage-uri.md) - - Internal Components - - [TiDB Backend Task Distributed Execution Framework](/tidb-distributed-execution-framework.md) - - [TiDB Global Sort](/tidb-global-sort.md) - FAQs - [FAQ Summary](/faq/faq-overview.md) - [TiDB FAQs](/faq/tidb-faq.md) @@ -1014,200 +920,4 @@ - [High Availability FAQs](/faq/high-availability-faq.md) - [High Reliability FAQs](/faq/high-reliability-faq.md) - [Backup and Restore FAQs](/faq/backup-and-restore-faq.md) -- Release Notes - - [All Releases](/releases/release-notes.md) - - [Release Timeline](/releases/release-timeline.md) - - [TiDB Versioning](/releases/versioning.md) - - [TiDB Installation Packages](/binary-package.md) - - v7.4 - - [7.4.0-DMR](/releases/release-7.4.0.md) - - v7.3 - - [7.3.0-DMR](/releases/release-7.3.0.md) - - v7.2 - - [7.2.0-DMR](/releases/release-7.2.0.md) - - v7.1 - - [7.1.2](/releases/release-7.1.2.md) - - [7.1.1](/releases/release-7.1.1.md) - - [7.1.0](/releases/release-7.1.0.md) - - v7.0 - - [7.0.0-DMR](/releases/release-7.0.0.md) - - v6.6 - - [6.6.0-DMR](/releases/release-6.6.0.md) - - v6.5 - - [6.5.5](/releases/release-6.5.5.md) - - [6.5.4](/releases/release-6.5.4.md) - - [6.5.3](/releases/release-6.5.3.md) - - [6.5.2](/releases/release-6.5.2.md) - - [6.5.1](/releases/release-6.5.1.md) - - [6.5.0](/releases/release-6.5.0.md) - - v6.4 - - [6.4.0-DMR](/releases/release-6.4.0.md) - - v6.3 - - [6.3.0-DMR](/releases/release-6.3.0.md) - - v6.2 - - [6.2.0-DMR](/releases/release-6.2.0.md) - - v6.1 - - [6.1.7](/releases/release-6.1.7.md) - - [6.1.6](/releases/release-6.1.6.md) - - [6.1.5](/releases/release-6.1.5.md) - - [6.1.4](/releases/release-6.1.4.md) - - [6.1.3](/releases/release-6.1.3.md) - - [6.1.2](/releases/release-6.1.2.md) - - [6.1.1](/releases/release-6.1.1.md) - - [6.1.0](/releases/release-6.1.0.md) - - v6.0 - - [6.0.0-DMR](/releases/release-6.0.0-dmr.md) - - v5.4 - - [5.4.3](/releases/release-5.4.3.md) - - [5.4.2](/releases/release-5.4.2.md) - - [5.4.1](/releases/release-5.4.1.md) - - [5.4.0](/releases/release-5.4.0.md) - - v5.3 - - [5.3.4](/releases/release-5.3.4.md) - - [5.3.3](/releases/release-5.3.3.md) - - [5.3.2](/releases/release-5.3.2.md) - - [5.3.1](/releases/release-5.3.1.md) - - [5.3.0](/releases/release-5.3.0.md) - - v5.2 - - [5.2.4](/releases/release-5.2.4.md) - - [5.2.3](/releases/release-5.2.3.md) - - [5.2.2](/releases/release-5.2.2.md) - - [5.2.1](/releases/release-5.2.1.md) - - [5.2.0](/releases/release-5.2.0.md) - - v5.1 - - [5.1.5](/releases/release-5.1.5.md) - - [5.1.4](/releases/release-5.1.4.md) - - [5.1.3](/releases/release-5.1.3.md) - - [5.1.2](/releases/release-5.1.2.md) - - [5.1.1](/releases/release-5.1.1.md) - - [5.1.0](/releases/release-5.1.0.md) - - v5.0 - - [5.0.6](/releases/release-5.0.6.md) - - [5.0.5](/releases/release-5.0.5.md) - - [5.0.4](/releases/release-5.0.4.md) - - [5.0.3](/releases/release-5.0.3.md) - - [5.0.2](/releases/release-5.0.2.md) - - [5.0.1](/releases/release-5.0.1.md) - - [5.0 GA](/releases/release-5.0.0.md) - - [5.0.0-rc](/releases/release-5.0.0-rc.md) - - v4.0 - - [4.0.16](/releases/release-4.0.16.md) - - [4.0.15](/releases/release-4.0.15.md) - - [4.0.14](/releases/release-4.0.14.md) - - [4.0.13](/releases/release-4.0.13.md) - - [4.0.12](/releases/release-4.0.12.md) - - [4.0.11](/releases/release-4.0.11.md) - - [4.0.10](/releases/release-4.0.10.md) - - [4.0.9](/releases/release-4.0.9.md) - - [4.0.8](/releases/release-4.0.8.md) - - [4.0.7](/releases/release-4.0.7.md) - - [4.0.6](/releases/release-4.0.6.md) - - [4.0.5](/releases/release-4.0.5.md) - - [4.0.4](/releases/release-4.0.4.md) - - [4.0.3](/releases/release-4.0.3.md) - - [4.0.2](/releases/release-4.0.2.md) - - [4.0.1](/releases/release-4.0.1.md) - - [4.0 GA](/releases/release-4.0-ga.md) - - [4.0.0-rc.2](/releases/release-4.0.0-rc.2.md) - - [4.0.0-rc.1](/releases/release-4.0.0-rc.1.md) - - [4.0.0-rc](/releases/release-4.0.0-rc.md) - - [4.0.0-beta.2](/releases/release-4.0.0-beta.2.md) - - [4.0.0-beta.1](/releases/release-4.0.0-beta.1.md) - - [4.0.0-beta](/releases/release-4.0.0-beta.md) - - v3.1 - - [3.1.2](/releases/release-3.1.2.md) - - [3.1.1](/releases/release-3.1.1.md) - - [3.1.0 GA](/releases/release-3.1.0-ga.md) - - [3.1.0-rc](/releases/release-3.1.0-rc.md) - - [3.1.0-beta.2](/releases/release-3.1.0-beta.2.md) - - [3.1.0-beta.1](/releases/release-3.1.0-beta.1.md) - - [3.1.0-beta](/releases/release-3.1.0-beta.md) - - v3.0 - - [3.0.20](/releases/release-3.0.20.md) - - [3.0.19](/releases/release-3.0.19.md) - - [3.0.18](/releases/release-3.0.18.md) - - [3.0.17](/releases/release-3.0.17.md) - - [3.0.16](/releases/release-3.0.16.md) - - [3.0.15](/releases/release-3.0.15.md) - - [3.0.14](/releases/release-3.0.14.md) - - [3.0.13](/releases/release-3.0.13.md) - - [3.0.12](/releases/release-3.0.12.md) - - [3.0.11](/releases/release-3.0.11.md) - - [3.0.10](/releases/release-3.0.10.md) - - [3.0.9](/releases/release-3.0.9.md) - - [3.0.8](/releases/release-3.0.8.md) - - [3.0.7](/releases/release-3.0.7.md) - - [3.0.6](/releases/release-3.0.6.md) - - [3.0.5](/releases/release-3.0.5.md) - - [3.0.4](/releases/release-3.0.4.md) - - [3.0.3](/releases/release-3.0.3.md) - - [3.0.2](/releases/release-3.0.2.md) - - [3.0.1](/releases/release-3.0.1.md) - - [3.0 GA](/releases/release-3.0-ga.md) - - [3.0.0-rc.3](/releases/release-3.0.0-rc.3.md) - - [3.0.0-rc.2](/releases/release-3.0.0-rc.2.md) - - [3.0.0-rc.1](/releases/release-3.0.0-rc.1.md) - - [3.0.0-beta.1](/releases/release-3.0.0-beta.1.md) - - [3.0.0-beta](/releases/release-3.0-beta.md) - - v2.1 - - [2.1.19](/releases/release-2.1.19.md) - - [2.1.18](/releases/release-2.1.18.md) - - [2.1.17](/releases/release-2.1.17.md) - - [2.1.16](/releases/release-2.1.16.md) - - [2.1.15](/releases/release-2.1.15.md) - - [2.1.14](/releases/release-2.1.14.md) - - [2.1.13](/releases/release-2.1.13.md) - - [2.1.12](/releases/release-2.1.12.md) - - [2.1.11](/releases/release-2.1.11.md) - - [2.1.10](/releases/release-2.1.10.md) - - [2.1.9](/releases/release-2.1.9.md) - - [2.1.8](/releases/release-2.1.8.md) - - [2.1.7](/releases/release-2.1.7.md) - - [2.1.6](/releases/release-2.1.6.md) - - [2.1.5](/releases/release-2.1.5.md) - - [2.1.4](/releases/release-2.1.4.md) - - [2.1.3](/releases/release-2.1.3.md) - - [2.1.2](/releases/release-2.1.2.md) - - [2.1.1](/releases/release-2.1.1.md) - - [2.1 GA](/releases/release-2.1-ga.md) - - [2.1 RC5](/releases/release-2.1-rc.5.md) - - [2.1 RC4](/releases/release-2.1-rc.4.md) - - [2.1 RC3](/releases/release-2.1-rc.3.md) - - [2.1 RC2](/releases/release-2.1-rc.2.md) - - [2.1 RC1](/releases/release-2.1-rc.1.md) - - [2.1 Beta](/releases/release-2.1-beta.md) - - v2.0 - - [2.0.11](/releases/release-2.0.11.md) - - [2.0.10](/releases/release-2.0.10.md) - - [2.0.9](/releases/release-2.0.9.md) - - [2.0.8](/releases/release-2.0.8.md) - - [2.0.7](/releases/release-2.0.7.md) - - [2.0.6](/releases/release-2.0.6.md) - - [2.0.5](/releases/release-2.0.5.md) - - [2.0.4](/releases/release-2.0.4.md) - - [2.0.3](/releases/release-2.0.3.md) - - [2.0.2](/releases/release-2.0.2.md) - - [2.0.1](/releases/release-2.0.1.md) - - [2.0](/releases/release-2.0-ga.md) - - [2.0 RC5](/releases/release-2.0-rc.5.md) - - [2.0 RC4](/releases/release-2.0-rc.4.md) - - [2.0 RC3](/releases/release-2.0-rc.3.md) - - [2.0 RC1](/releases/release-2.0-rc.1.md) - - [1.1 Beta](/releases/release-1.1-beta.md) - - [1.1 Alpha](/releases/release-1.1-alpha.md) - - v1.0 - - [1.0.8](/releases/release-1.0.8.md) - - [1.0.7](/releases/release-1.0.7.md) - - [1.0.6](/releases/release-1.0.6.md) - - [1.0.5](/releases/release-1.0.5.md) - - [1.0.4](/releases/release-1.0.4.md) - - [1.0.3](/releases/release-1.0.3.md) - - [1.0.2](/releases/release-1.0.2.md) - - [1.0.1](/releases/release-1.0.1.md) - - [1.0](/releases/release-1.0-ga.md) - - [Pre-GA](/releases/release-pre-ga.md) - - [RC4](/releases/release-rc.4.md) - - [RC3](/releases/release-rc.3.md) - - [RC2](/releases/release-rc.2.md) - - [RC1](/releases/release-rc.1.md) - [Glossary](/glossary.md) diff --git a/_docHome.md b/_docHome.md index c9db9fad37ad6..0e9cb975b0125 100644 --- a/_docHome.md +++ b/_docHome.md @@ -3,9 +3,10 @@ title: PingCAP Documentation hide_sidebar: true hide_commit: true hide_leftNav: true +summary: TiDB Documentation provides how-to guides and references for using TiDB Cloud and TiDB Self-Managed, including data migration and application building. TiDB Cloud is a fully-managed Database-as-a-Service, offering easy access to the power of a cloud-native, distributed SQL database. TiDB is an open-source distributed SQL database with MySQL compatibility, horizontal scalability, and high availability. Developers can access documentation for application development and explore additional resources such as TiDB Playground, PingCAP Education, and community engagement opportunities. --- - + @@ -13,33 +14,41 @@ TiDB Cloud is a fully-managed Database-as-a-Service (DBaaS) that brings everythi - + -View the documentation for TiDB Cloud. +Learn what TiDB Cloud is as an easy-to-use database and its key features. - + Guide for an easy way to get started with TiDB Cloud. - + -Helps you quickly complete a Proof of Concept (PoC) with TiDB Cloud. +Connect your application with the languages and frameworks you prefer. - + + +Explore native support of Vector Search in {{{ .starter }}} to build your AI application. + + -Get the power of a cloud-native, distributed SQL database built for real-time analytics in a fully-managed service. + + +Planned features and releases for TiDB Cloud. + + -Try Free + - + -TiDB is an open-source distributed SQL database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. It is MySQL compatible and features horizontal scalability, strong consistency, and high availability. You can deploy TiDB in a self-hosted environment or in the cloud. +TiDB is an open-source distributed SQL database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. It is MySQL compatible and features horizontal scalability, strong consistency, and high availability. You can deploy TiDB in a self-hosted environment or on the cloud. - + -View the documentation for TiDB. +Learn what is TiDB Self-Managed and its key features. - + -Walks you through the quickest way to get started with TiDB. +Walks you through the quickest way to get started with TiDB Self-Managed. - + Learn how to deploy TiDB locally in a production environment. - - -The open-source TiDB platform is released under the Apache 2.0 license and is supported by the community. - -Download - - - - + - - - - -Documentation for TiDB application developers. +For application developers using TiDB Self-Managed. - + -Documentation for TiDB Cloud application developers. +TiDB is highly compatible with the MySQL protocol and the common features and syntax of MySQL 5.7 and MySQL 8.0. +The open-source TiDB platform is released under the Apache 2.0 license and is supported by the community. [View on GitHub](https://github.com/pingcap/tidb) + - + - + -Experience the capabilities of TiDB WITHOUT registration. +Learn TiDB Cloud and TiDB Self-Managed through well-designed online courses and instructor-led training. - + -Learn TiDB and TiDB Cloud through well-designed online courses and instructor-led training. +Read great articles about TiDB Cloud and TiDB Self-Managed. - + -Join us on Discord or become a contributor. +Learn about events hosted by PingCAP and the community. - + -Read great articles about TiDB and TiDB Cloud. +Download eBooks and papers. - + Watch a compilation of short videos describing TiDB and various use cases. - + -Learn about events hosted by PingCAP and the community. +A powerful insight tool that provides in-depth analysis of any GitHub repository, powered by TiDB Cloud. - + -Download eBooks and papers. +Experience the capabilities of TiDB without registration. - + -A powerful insight tool that analyzes any GitHub repository in depth, powered by TiDB Cloud. - - - - - -Let's work together to improve the documentation! +Join us on Discord or become a contributor. diff --git a/_index.md b/_index.md index 0d249ee544799..845e886064529 100644 --- a/_index.md +++ b/_index.md @@ -1,11 +1,11 @@ --- title: TiDB Introduction -aliases: ["/docs/dev/", "/docs/dev/adopters/", "/tidb/dev/adopters"] hide_sidebar: true hide_commit: true +summary: TiDB is an open-source distributed SQL database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. The guide provides information on features, TiFlash, development, deployment, migration, maintenance, monitoring, tuning, tools, and references. It covers everything from quick start to advanced configurations and tools for TiDB. --- - + @@ -66,21 +65,6 @@ This section gives the alert rules for the TiDB component. Same as [`TiDB_schema_error`](#tidb_schema_error). -#### `TiDB_monitor_keep_alive` - -* Alert rule: - - `increase(tidb_monitor_keep_alive_total[10m]) < 100` - -* Description: - - Indicates whether the TiDB process still exists. If the number of times for `tidb_monitor_keep_alive_total` increases less than 100 in 10 minutes, the TiDB process might already exit and an alert is triggered. - -* Solution: - - * Check whether the TiDB process is out of memory. - * Check whether the machine has restarted. - ### Critical-level alerts #### `TiDB_server_panic_total` @@ -121,7 +105,7 @@ This section gives the alert rules for the TiDB component. * Description: - The latency of handling a request in TiDB. If the ninety-ninth percentile latency exceeds 1 second, an alert is triggered. + The latency of handling a request in TiDB. The response time for 99% of requests should be within 1 second; otherwise, an alert is triggered. * Solution: @@ -578,11 +562,13 @@ This section gives the alert rules for the TiKV component. * Alert rule: - `sum(rate(tikv_thread_cpu_seconds_total{name=~"raftstore_.*"}[1m])) by (instance, name) > 1.6` + `sum(rate(tikv_thread_cpu_seconds_total{name=~"raftstore_.*"}[1m])) by (instance) > 1.6` * Description: - The pressure on the Raftstore thread is too high. + This rule monitors CPU usage by Raftstore. If the value is high, it indicates pressure on Raftstore threads is heavy. + + The alert threshold is 80% of the [`raftstore.store-pool-size`](/tikv-configuration-file.md#store-pool-size) value. `raftstore.store-pool-size` is 2 by default, so the alert threshold is 1.6. * Solution: @@ -620,8 +606,8 @@ This section gives the alert rules for the TiKV component. * Solution: - 1. View the scheduler command duration in the Scheduler-All monitor and see which command is most time-consuming; - 2. View the scheduler scan details in the Scheduler-All monitor and see whether `total` and `process` match. If they differ a lot, there are many invalid scans. You can also see whether there is `over seek bound`. If there is too much, it indicates GC does not work in time; + 1. Identify the most time-consuming command by viewing the scheduler command duration in the `Scheduler` and `Scheduler-${cmd}` (`${cmd}` is the write command to query) monitors; + 2. Check the `total` and `process` values in the scheduler scan details of the `Scheduler` and `Scheduler-${cmd}` monitors and see whether `total` and `process` match. 3. View the storage async snapshot/write duration in the Storage monitor and see whether the Raft operation is performed in time. #### `TiKV_thread_apply_worker_cpu_seconds` diff --git a/as-of-timestamp.md b/as-of-timestamp.md index daaf9ba060ed8..76872ff5c13bc 100644 --- a/as-of-timestamp.md +++ b/as-of-timestamp.md @@ -23,7 +23,7 @@ You can use the `AS OF TIMESTAMP` clause in the following three ways: If you want to specify an exact point of time, you can set a datetime value or use a time function in the `AS OF TIMESTAMP` clause. The format of datetime is like "2016-10-08 16:45:26.999", with millisecond as the minimum time unit, but for most of the time, the time unit of second is enough for specifying a datetime, such as "2016-10-08 16:45:26". You can also get the current time to the millisecond using the `NOW(3)` function. If you want to read the data of several seconds ago, it is **recommended** to use an expression such as `NOW() - INTERVAL 10 SECOND`. -If you want to specify a time range, you can use the `TIDB_BOUNDED_STALENESS()` function in the clause. When this function is used, TiDB selects a suitable timestamp within the specified time range. "Suitable" means there are no transactions that start before this timestamp and have not been committed on the accessed replica, that is, TiDB can perform read operations on the accessed replica and the read operations are not blocked. You need to use `TIDB_BOUNDED_STALENESS(t1, t2)` to call this function. `t1` and `t2` are the two ends of the time range, which can be specified using either datetime values or time functions. +If you want to specify a time range, you can use the [`TIDB_BOUNDED_STALENESS()`](/functions-and-operators/tidb-functions.md#tidb_bounded_staleness) function in the clause. When this function is used, TiDB selects a suitable timestamp within the specified time range. "Suitable" means there are no transactions that start before this timestamp and have not been committed on the accessed replica, that is, TiDB can perform read operations on the accessed replica and the read operations are not blocked. You need to use `TIDB_BOUNDED_STALENESS(t1, t2)` to call this function. `t1` and `t2` are the two ends of the time range, which can be specified using either datetime values or time functions. Here are some examples of the `AS OF TIMESTAMP` clause: diff --git a/auto-increment.md b/auto-increment.md index a227a8e5e1e22..b61f49c874095 100644 --- a/auto-increment.md +++ b/auto-increment.md @@ -1,7 +1,6 @@ --- title: AUTO_INCREMENT summary: Learn the `AUTO_INCREMENT` column attribute of TiDB. -aliases: ['/docs/dev/auto-increment/'] --- # AUTO_INCREMENT @@ -304,46 +303,116 @@ After the value `2030000` is inserted, the next value is `2060001`. This jump in In earlier versions of TiDB, the cache size of the auto-increment ID was transparent to users. Starting from v3.0.14, v3.1.2, and v4.0.rc-2, TiDB has introduced the `AUTO_ID_CACHE` table option to allow users to set the cache size for allocating the auto-increment ID. ```sql -mysql> CREATE TABLE t(a int AUTO_INCREMENT key) AUTO_ID_CACHE 100; +CREATE TABLE t(a int AUTO_INCREMENT key) AUTO_ID_CACHE 100; Query OK, 0 rows affected (0.02 sec) -mysql> INSERT INTO t values(); +INSERT INTO t values(); Query OK, 1 row affected (0.00 sec) -Records: 1 Duplicates: 0 Warnings: 0 -mysql> SELECT * FROM t; +SELECT * FROM t; +---+ | a | +---+ | 1 | +---+ 1 row in set (0.01 sec) + +SHOW CREATE TABLE t; ++-------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Table | Create Table | ++-------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| t | CREATE TABLE `t` ( + `a` int(11) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=101 /*T![auto_id_cache] AUTO_ID_CACHE=100 */ | ++-------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +1 row in set (0.00 sec) ``` -At this time, if you invalidate the auto-increment cache of this column and redo the implicit insertion, the result is as follows: +At this time, if you restart TiDB, the auto-increment ID cache will be lost, and new insert operations will allocate IDs starting from a higher value beyond the previously cached range. ```sql -mysql> DELETE FROM t; -Query OK, 1 row affected (0.01 sec) - -mysql> RENAME TABLE t to t1; -Query OK, 0 rows affected (0.01 sec) - -mysql> INSERT INTO t1 values() +INSERT INTO t VALUES(); Query OK, 1 row affected (0.00 sec) -mysql> SELECT * FROM t; +SELECT * FROM t; +-----+ | a | +-----+ +| 1 | | 101 | +-----+ -1 row in set (0.00 sec) +2 rows in set (0.01 sec) +``` + +The newly allocated value is `101`. This shows that the size of cache for allocating auto-increment IDs is `100`. + +In addition, when the length of consecutive IDs in a batch `INSERT` statement exceeds the length of `AUTO_ID_CACHE`, TiDB increases the cache size accordingly to ensure that the statement can insert data properly. + +### Clear the auto-increment ID cache + +In some scenarios, you might need to clear the auto-increment ID cache to ensure data consistency. For example: + + + +- In the scenario of incremental replication using [Data Migration (DM)](/dm/dm-overview.md), once the replication is complete, data writing to the downstream TiDB switches from DM to your application's write operations. Meanwhile, the ID writing mode of the auto-increment column usually switches from explicit insertion to implicit allocation. + + + + +- In the scenario of incremental replication using the [Data Migration](/tidb-cloud/migrate-incremental-data-from-mysql-using-data-migration.md) feature, once the replication is complete, data writing to the downstream TiDB switches from DM to your application's write operations. Meanwhile, the ID writing mode of the auto-increment column usually switches from explicit insertion to implicit allocation. + + + +- When your application involves both explicit ID insertion and implicit ID allocation, you need to clear the auto-increment ID cache to avoid conflicts between future implicitly allocated IDs and previously explicitly inserted IDs, which could result in primary key conflict errors. For more information, see [Uniqueness](/auto-increment.md#uniqueness). + +To clear the auto-increment ID cache on all TiDB nodes in the cluster, you can execute the `ALTER TABLE` statement with `AUTO_INCREMENT = 0`. For example: + +```sql +CREATE TABLE t(a int AUTO_INCREMENT key) AUTO_ID_CACHE 100; +Query OK, 0 rows affected (0.02 sec) + +INSERT INTO t VALUES(); +Query OK, 1 row affected (0.02 sec) + +INSERT INTO t VALUES(50); +Query OK, 1 row affected (0.00 sec) + +SELECT * FROM t; ++----+ +| a | ++----+ +| 1 | +| 50 | ++----+ +2 rows in set (0.01 sec) ``` -The re-assigned value is `101`. This shows that the size of cache for allocating the auto-increment ID is `100`. +```sql +ALTER TABLE t AUTO_INCREMENT = 0; +Query OK, 0 rows affected, 1 warning (0.07 sec) + +SHOW WARNINGS; ++---------+------+-------------------------------------------------------------------------+ +| Level | Code | Message | ++---------+------+-------------------------------------------------------------------------+ +| Warning | 1105 | Can't reset AUTO_INCREMENT to 0 without FORCE option, using 101 instead | ++---------+------+-------------------------------------------------------------------------+ +1 row in set (0.01 sec) + +INSERT INTO t VALUES(); +Query OK, 1 row affected (0.02 sec) -In addition, when the length of consecutive IDs in a batch `INSERT` statement exceeds the length of `AUTO_ID_CACHE`, TiDB increases the cache size accordingly to ensure that the statement can be inserted properly. +SELECT * FROM t; ++-----+ +| a | ++-----+ +| 1 | +| 50 | +| 101 | ++-----+ +3 rows in set (0.01 sec) +``` ### Auto-increment step size and offset diff --git a/auto-random.md b/auto-random.md index 5a3e2e57065e8..ba72b10c9235f 100644 --- a/auto-random.md +++ b/auto-random.md @@ -1,7 +1,6 @@ --- title: AUTO_RANDOM summary: Learn the AUTO_RANDOM attribute. -aliases: ['/docs/dev/auto-random/','/docs/dev/reference/sql/attributes/auto-random/'] --- # AUTO_RANDOM New in v3.1.0 @@ -47,7 +46,7 @@ When you execute an `INSERT` statement: - If you do not explicitly specify the value of the `AUTO_RANDOM` column, TiDB generates a random value and inserts it into the table. ```sql -tidb> CREATE TABLE t (a BIGINT PRIMARY KEY AUTO_RANDOM, b VARCHAR(255)); +tidb> CREATE TABLE t (a BIGINT PRIMARY KEY AUTO_RANDOM, b VARCHAR(255)) /*T! PRE_SPLIT_REGIONS=2 */ ; Query OK, 0 rows affected, 1 warning (0.01 sec) tidb> INSERT INTO t(a, b) VALUES (1, 'string'); @@ -76,6 +75,29 @@ tidb> SELECT * FROM t; | 4899916394579099651 | string3 | +---------------------+---------+ 3 rows in set (0.00 sec) + +tidb> SHOW CREATE TABLE t; ++-------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Table | Create Table | ++-------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| t | CREATE TABLE `t` ( + `a` bigint(20) NOT NULL /*T![auto_rand] AUTO_RANDOM(5) */, + `b` varchar(255) DEFAULT NULL, + PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T! PRE_SPLIT_REGIONS=2 */ | ++-------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +1 row in set (0.00 sec) + +tidb> SHOW TABLE t REGIONS; ++-----------+-----------------------------+-----------------------------+-----------+-----------------+---------------------+------------+---------------+------------+----------------------+------------------+------------------------+------------------+ +| REGION_ID | START_KEY | END_KEY | LEADER_ID | LEADER_STORE_ID | PEERS | SCATTERING | WRITTEN_BYTES | READ_BYTES | APPROXIMATE_SIZE(MB) | APPROXIMATE_KEYS | SCHEDULING_CONSTRAINTS | SCHEDULING_STATE | ++-----------+-----------------------------+-----------------------------+-----------+-----------------+---------------------+------------+---------------+------------+----------------------+------------------+------------------------+------------------+ +| 62798 | t_158_ | t_158_r_2305843009213693952 | 62810 | 28 | 62811, 62812, 62810 | 0 | 151 | 0 | 1 | 0 | | | +| 62802 | t_158_r_2305843009213693952 | t_158_r_4611686018427387904 | 62803 | 1 | 62803, 62804, 62805 | 0 | 39 | 0 | 1 | 0 | | | +| 62806 | t_158_r_4611686018427387904 | t_158_r_6917529027641081856 | 62813 | 4 | 62813, 62814, 62815 | 0 | 160 | 0 | 1 | 0 | | | +| 9289 | t_158_r_6917529027641081856 | 78000000 | 48268 | 1 | 48268, 58951, 62791 | 0 | 10628 | 43639 | 2 | 7999 | | | ++-----------+-----------------------------+-----------------------------+-----------+-----------------+---------------------+------------+---------------+------------+----------------------+------------------+------------------------+------------------+ +4 rows in set (0.00 sec) ``` The `AUTO_RANDOM(S, R)` column value automatically assigned by TiDB has a total of 64 bits: @@ -83,16 +105,26 @@ The `AUTO_RANDOM(S, R)` column value automatically assigned by TiDB has a total - `S` is the number of shard bits. The value ranges from `1` to `15`. The default value is `5`. - `R` is the total length of the automatic allocation range. The value ranges from `32` to `64`. The default value is `64`. -The structure of an `AUTO_RANDOM` value is as follows: +The structure of an `AUTO_RANDOM` value with a signed bit is as follows: + +| Signed bit | Reserved bits | Shard bits | Auto-increment bits | +|---------|-------------|--------|--------------| +| 1 bit | `64-R` bits | `S` bits | `R-1-S` bits | -| Total number of bits | Sign bit | Reserved bits | Shard bits | Auto-increment bits | -|---------|---------|-------------|--------|--------------| -| 64 bits | 0/1 bit | (64-R) bits | S bits | (R-1-S) bits | +The structure of an `AUTO_RANDOM` value without a signed bit is as follows: -- The length of the sign bit is determined by the existence of an `UNSIGNED` attribute. If there is an `UNSIGNED` attribute, the length is `0`. Otherwise, the length is `1`. +| Reserved bits | Shard bits | Auto-increment bits | +|-------------|--------|--------------| +| `64-R` bits | `S` bits | `R-S` bits | + +- Whether an `AUTO_RANDOM` column has a signed bit depends on whether the column has the `UNSIGNED` attribute. A column without the `UNSIGNED` attribute has one signed bit, while a column with the `UNSIGNED` attribute has no signed bit. +- Values implicitly allocated to an `AUTO_RANDOM` column are always positive. For columns with a signed bit, the signed bit of implicitly allocated values is always `0`. This rule does not apply to explicitly inserted values. - The length of the reserved bits is `64-R`. The reserved bits are always `0`. +- The auto-increment bits must be at least 27 bits. For signed columns, `R-1-S >= 27` must be satisfied. For unsigned columns, `R-S >= 27` must be satisfied. - The content of the shard bits is obtained by calculating the hash value of the starting time of the current transaction. To use a different length of shard bits (such as 10), you can specify `AUTO_RANDOM(10)` when creating the table. - The value of the auto-increment bits is stored in the storage engine and allocated sequentially. Each time a new value is allocated, the value is incremented by 1. The auto-increment bits ensure that the values of `AUTO_RANDOM` are unique globally. When the auto-increment bits are exhausted, an error `Failed to read auto-increment value from storage engine` is reported when the value is allocated again. +- Value range: the maximum number of bits for the final generated value = shard bits + auto-increment bits. The range of a signed column is `[-(2^(R-1))+1, (2^(R-1))-1]`, and the range of an unsigned column is `[0, (2^R)-1]`. +- You can use `AUTO_RANDOM` with `PRE_SPLIT_REGIONS`. When a table is created successfully, `PRE_SPLIT_REGIONS` pre-splits data in the table into the number of Regions as specified by `2^(PRE_SPLIT_REGIONS)`. > **Note:** > @@ -104,7 +136,7 @@ The structure of an `AUTO_RANDOM` value is as follows: > Selection of range (`R`): > > - Typically, the `R` parameter needs to be set when the numeric type of the application cannot represent a full 64-bit integer. -> - For example, the range of JSON number is `[-2^53+1, 2^53-1]`. TiDB can easily assign an integer outside this range to a column of `AUTO_RANDOM(5)`, causing unexpected behaviors when the application reads the column. In this case, you can replace `AUTO_RANDOM(5)` with `AUTO_RANDOM(5, 54)` and TiDB does not assign an integer greater than `9007199254740991` (2^53-1) to the column. +> - For example, the range of JSON number is `[-(2^53)+1, (2^53)-1]`. TiDB can easily assign an integer beyond this range to a column defined as `AUTO_RANDOM(5)`, causing unexpected behaviors when the application reads the column. In such cases, you can replace `AUTO_RANDOM(5)` with `AUTO_RANDOM(5, 54)` for signed columns, and replace `AUTO_RANDOM(5)` with `AUTO_RANDOM(5, 53)` for unsigned columns, ensuring that TiDB does not assign integers greater than `9007199254740991` (2^53-1) to the column. Values allocated implicitly to the `AUTO_RANDOM` column affect `last_insert_id()`. To get the ID that TiDB last implicitly allocates, you can use the `SELECT last_insert_id ()` statement. @@ -128,6 +160,10 @@ The output is as follows: 1 row in set (0.00 sec) ``` +## Implicit allocation rules of IDs + +TiDB implicitly allocates values to `AUTO_RANDOM` columns similarly to `AUTO_INCREMENT` columns. They are also controlled by the session-level system variables [`auto_increment_increment`](/system-variables.md#auto_increment_increment) and [`auto_increment_offset`](/system-variables.md#auto_increment_offset). The auto-increment bits (ID) of implicitly allocated values conform to the equation `(ID - auto_increment_offset) % auto_increment_increment == 0`. + ## Restrictions Pay attention to the following restrictions when you use `AUTO_RANDOM`: diff --git a/backup-and-restore-using-dumpling-lightning.md b/backup-and-restore-using-dumpling-lightning.md index fa4942a0c17af..640969e2a47c8 100644 --- a/backup-and-restore-using-dumpling-lightning.md +++ b/backup-and-restore-using-dumpling-lightning.md @@ -7,20 +7,22 @@ summary: Learn how to use Dumpling and TiDB Lightning to back up and restore ful This document introduces how to use Dumpling and TiDB Lightning to back up and restore full data of TiDB. -If you need to back up a small amount of data (for example, less than 50 GB) and do not require high backup speed, you can use [Dumpling](/dumpling-overview.md) to export data from the TiDB database and then use [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md) to import the data into another TiDB database. For more information about backup and restore, see [TiDB Backup & Restore Overview](/br/backup-and-restore-overview.md). +If you need to back up a small amount of data (for example, less than 50 GiB) and do not require high backup speed, you can use [Dumpling](/dumpling-overview.md) to export data from the TiDB database and then use [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md) to restore the data into another TiDB database. + +If you need to back up larger databases, the recommended method is to use [BR](/br/backup-and-restore-overview.md). Note that Dumpling can be used to export large databases, but BR is a better tool for that. ## Requirements -- Install and start Dumpling: +- Install Dumpling: ```shell - tiup install dumpling && tiup dumpling + tiup install dumpling ``` -- Install and start TiDB Lightning: +- Install TiDB Lightning: ```shell - tiup install tidb lightning && tiup tidb lightning + tiup install tidb-lightning ``` - [Grant the source database privileges required for Dumpling](/dumpling-overview.md#export-data-from-tidb-or-mysql) @@ -41,14 +43,35 @@ If you need to save data of one backup task to the local disk, note the followin - Dumpling requires a disk space that can store the whole data source (or to store all upstream tables to be exported). To calculate the required space, see [Downstream storage space requirements](/tidb-lightning/tidb-lightning-requirements.md#storage-space-of-the-target-database). - During the import, TiDB Lightning needs temporary space to store the sorted key-value pairs. The disk space should be enough to hold the largest single table from the data source. -**Note**: It is difficult to calculate the exact data volume exported by Dumpling from MySQL, but you can estimate the data volume by using the following SQL statement to summarize the `data-length` field in the `information_schema.tables` table: +**Note**: It is difficult to calculate the exact data volume exported by Dumpling from MySQL, but you can estimate the data volume by using the following SQL statement to summarize the `DATA_LENGTH` field in the `information_schema.tables` table: ```sql -/* Calculate the size of all schemas, in MiB. Replace ${schema_name} with your schema name. */ -SELECT table_schema,SUM(data_length)/1024/1024 AS data_length,SUM(index_length)/1024/1024 AS index_length,SUM(data_length+index_length)/1024/1024 AS SUM FROM information_schema.tables WHERE table_schema = "${schema_name}" GROUP BY table_schema; - -/* Calculate the size of the largest table, in MiB. Replace ${schema_name} with your schema name. */ -SELECT table_name,table_schema,SUM(data_length)/1024/1024 AS data_length,SUM(index_length)/1024/1024 AS index_length,SUM(data_length+index_length)/1024/1024 AS SUM from information_schema.tables WHERE table_schema = "${schema_name}" GROUP BY table_name,table_schema ORDER BY SUM DESC LIMIT 5; +-- Calculate the size of all schemas +SELECT + TABLE_SCHEMA, + FORMAT_BYTES(SUM(DATA_LENGTH)) AS 'Data Size', + FORMAT_BYTES(SUM(INDEX_LENGTH)) 'Index Size' +FROM + information_schema.tables +GROUP BY + TABLE_SCHEMA; + +-- Calculate the 5 largest tables +SELECT + TABLE_NAME, + TABLE_SCHEMA, + FORMAT_BYTES(SUM(data_length)) AS 'Data Size', + FORMAT_BYTES(SUM(index_length)) AS 'Index Size', + FORMAT_BYTES(SUM(data_length+index_length)) AS 'Total Size' +FROM + information_schema.tables +GROUP BY + TABLE_NAME, + TABLE_SCHEMA +ORDER BY + SUM(DATA_LENGTH+INDEX_LENGTH) DESC +LIMIT + 5; ``` ### Disk space for the target TiKV cluster diff --git a/basic-features.md b/basic-features.md index 6127851320cd3..f92d8b0617ce3 100644 --- a/basic-features.md +++ b/basic-features.md @@ -1,7 +1,6 @@ --- title: TiDB Features summary: Learn about the feature overview of TiDB. -aliases: ['/docs/dev/basic-features/','/tidb/dev/experimental-features-4.0/'] --- # TiDB Features @@ -22,235 +21,234 @@ You can try out TiDB features on [TiDB Playground](https://play.tidbcloud.com/?u ## Data types, functions, and operators -| Data types, functions, and operators | 7.4 | 7.3 | 7.2 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Numeric types](/data-type-numeric.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Date and time types](/data-type-date-and-time.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [String types](/data-type-string.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [JSON type](/data-type-json.md) | Y | Y | Y | Y | Y | E | E | E | E | E | E | E | -| [Control flow functions](/functions-and-operators/control-flow-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [String functions](/functions-and-operators/string-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Numeric functions and operators](/functions-and-operators/numeric-functions-and-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Date and time functions](/functions-and-operators/date-and-time-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Bit functions and operators](/functions-and-operators/bit-functions-and-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Cast functions and operators](/functions-and-operators/cast-functions-and-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Encryption and compression functions](/functions-and-operators/encryption-and-compression-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Information functions](/functions-and-operators/information-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [JSON functions](/functions-and-operators/json-functions.md) | Y | Y | Y | Y | Y | E | E | E | E | E | E | E | -| [Aggregation functions](/functions-and-operators/aggregate-group-by-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Window functions](/functions-and-operators/window-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Miscellaneous functions](/functions-and-operators/miscellaneous-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Operators](/functions-and-operators/operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Character sets and collations](/character-set-and-collation.md) [^1] | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [User-level lock](/functions-and-operators/locking-functions.md) | Y | Y | Y | Y | Y | Y | N | N | N | N | N | N | +| Data types, functions, and operators | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [Numeric types](/data-type-numeric.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Date and time types](/data-type-date-and-time.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [String types](/data-type-string.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [JSON type](/data-type-json.md) | Y | Y | Y | E | E | E | E | E | E | E | +| [Control flow functions](/functions-and-operators/control-flow-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [String functions](/functions-and-operators/string-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Numeric functions and operators](/functions-and-operators/numeric-functions-and-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Date and time functions](/functions-and-operators/date-and-time-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Bit functions and operators](/functions-and-operators/bit-functions-and-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Cast functions and operators](/functions-and-operators/cast-functions-and-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Encryption and compression functions](/functions-and-operators/encryption-and-compression-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Information functions](/functions-and-operators/information-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [JSON functions](/functions-and-operators/json-functions.md) | Y | Y | Y | E | E | E | E | E | E | E | +| [Aggregation functions](/functions-and-operators/aggregate-group-by-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Window functions](/functions-and-operators/window-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Miscellaneous functions](/functions-and-operators/miscellaneous-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Operators](/functions-and-operators/operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Character sets and collations](/character-set-and-collation.md) [^1] | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [User-level lock](/functions-and-operators/locking-functions.md) | Y | Y | Y | Y | N | N | N | N | N | N | ## Indexing and constraints -| Indexing and constraints | 7.4 | 7.3 | 7.2 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Expression indexes](/sql-statements/sql-statement-create-index.md#expression-index) [^2] | Y | Y | Y | Y | Y | E | E | E | E | E | E | E | -| [Columnar storage (TiFlash)](/tiflash/tiflash-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Use FastScan to accelerate queries in OLAP scenarios](/tiflash/use-fastscan.md) | Y | Y | Y | Y | E | N | N | N | N | N | N | N | -| [RocksDB engine](/storage-engine/rocksdb-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Titan plugin](/storage-engine/titan-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Titan Level Merge](/storage-engine/titan-configuration.md#level-merge-experimental) | E | E | E | E | E | E | E | E | E | E | E | E | -| [Use buckets to improve scan concurrency](/tune-region-performance.md#use-bucket-to-increase-concurrency) | E | E | E | E | E | E | N | N | N | N | N | N | -| [Invisible indexes](/sql-statements/sql-statement-add-index.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | -| [Composite `PRIMARY KEY`](/constraints.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [`CHECK` constraints](/constraints.md#check) | Y | Y | Y | N | N | N | N | N | N | N | N | N | -| [Unique indexes](/constraints.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Clustered index on integer `PRIMARY KEY`](/clustered-indexes.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Clustered index on composite or non-integer key](/clustered-indexes.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | -| [Multi-valued indexes](/sql-statements/sql-statement-create-index.md#multi-valued-indexes) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [Foreign key](/constraints.md#foreign-key) | E | E | E | E | N | N | N | N | N | N | N | N | -| [TiFlash late materialization](/tiflash/tiflash-late-materialization.md) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | +| Indexing and constraints | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [Expression indexes](/sql-statements/sql-statement-create-index.md#expression-index) [^2] | Y | Y | Y | E | E | E | E | E | E | E | +| [Columnar storage (TiFlash)](/tiflash/tiflash-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Use FastScan to accelerate queries in OLAP scenarios](/tiflash/use-fastscan.md) | Y | Y | E | N | N | N | N | N | N | N | +| [RocksDB engine](/storage-engine/rocksdb-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Titan plugin](/storage-engine/titan-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Titan Level Merge](/storage-engine/titan-configuration.md#level-merge-experimental) | E | E | E | E | E | E | E | E | E | E | +| [Use buckets to improve scan concurrency](/tune-region-performance.md#use-bucket-to-increase-concurrency) | E | E | E | E | N | N | N | N | N | N | +| [Invisible indexes](/sql-statements/sql-statement-create-index.md#invisible-index) | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | +| [Composite `PRIMARY KEY`](/constraints.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [`CHECK` constraints](/constraints.md#check) | Y | N | N | N | N | N | N | N | N | N | +| [Unique indexes](/constraints.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Clustered index on integer `PRIMARY KEY`](/clustered-indexes.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Clustered index on composite or non-integer key](/clustered-indexes.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | +| [Multi-valued indexes](/sql-statements/sql-statement-create-index.md#multi-valued-indexes) | Y | Y | N | N | N | N | N | N | N | N | +| [Foreign key](/constraints.md#foreign-key) | E | E | N | N | N | N | N | N | N | N | +| [TiFlash late materialization](/tiflash/tiflash-late-materialization.md) | Y | Y | N | N | N | N | N | N | N | N | ## SQL statements -| SQL statements [^3] | 7.4 | 7.3 | 7.2 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| Basic `SELECT`, `INSERT`, `UPDATE`, `DELETE`, `REPLACE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| `INSERT ON DUPLICATE KEY UPDATE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| `LOAD DATA INFILE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| `SELECT INTO OUTFILE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| `INNER JOIN`, LEFT\|RIGHT [OUTER] JOIN | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| `UNION`, `UNION ALL` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [`EXCEPT` and `INTERSECT` operators](/functions-and-operators/set-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | -| `GROUP BY`, `ORDER BY` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Window Functions](/functions-and-operators/window-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Common Table Expressions (CTE)](/sql-statements/sql-statement-with.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | N | -| `START TRANSACTION`, `COMMIT`, `ROLLBACK` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [`EXPLAIN`](/sql-statements/sql-statement-explain.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [User-defined variables](/user-defined-variables.md) | E | E | E | E | E | E | E | E | E | E | E | E | -| [`BATCH [ON COLUMN] LIMIT INTEGER DELETE`](/sql-statements/sql-statement-batch.md) | Y | Y | Y | Y | Y | Y | N | N | N | N | N | N | -| [`BATCH [ON COLUMN] LIMIT INTEGER INSERT/UPDATE/REPLACE`](/sql-statements/sql-statement-batch.md) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [`ALTER TABLE ... COMPACT`](/sql-statements/sql-statement-alter-table-compact.md) | Y | Y | Y | Y | Y | E | N | N | N | N | N | N | -| [Table Lock](/sql-statements/sql-statement-lock-tables-and-unlock-tables.md) | E | E | E | E | E | E | E | E | E | E | E | E | -| [TiFlash Query Result Materialization](/tiflash/tiflash-results-materialization.md) | Y| Y | Y | Y | E | N | N | N | N | N | N | N | +| SQL statements [^3] | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| Basic `SELECT`, `INSERT`, `UPDATE`, `DELETE`, `REPLACE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `INSERT ON DUPLICATE KEY UPDATE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `LOAD DATA INFILE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `SELECT INTO OUTFILE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `INNER JOIN`, LEFT\|RIGHT [OUTER] JOIN | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `UNION`, `UNION ALL` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [`EXCEPT` and `INTERSECT` operators](/functions-and-operators/set-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | +| `GROUP BY`, `ORDER BY` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Window Functions](/functions-and-operators/window-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Common Table Expressions (CTE)](/sql-statements/sql-statement-with.md) | Y | Y | Y | Y | Y | Y | Y | Y | N | N | +| `START TRANSACTION`, `COMMIT`, `ROLLBACK` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [`EXPLAIN`](/sql-statements/sql-statement-explain.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [User-defined variables](/user-defined-variables.md) | E | E | E | E | E | E | E | E | E | E | +| [`BATCH [ON COLUMN] LIMIT INTEGER DELETE`](/sql-statements/sql-statement-batch.md) | Y | Y | Y | Y | N | N | N | N | N | N | +| [`BATCH [ON COLUMN] LIMIT INTEGER INSERT/UPDATE/REPLACE`](/sql-statements/sql-statement-batch.md) | Y | Y | Y | N | N | N | N | N | N | N | +| [`ALTER TABLE ... COMPACT`](/sql-statements/sql-statement-alter-table-compact.md) | Y | Y | Y | E | N | N | N | N | N | N | +| [Table Lock](/sql-statements/sql-statement-lock-tables-and-unlock-tables.md) | E | E | E | E | E | E | E | E | E | E | +| [TiFlash Query Result Materialization](/tiflash/tiflash-results-materialization.md) | Y | Y | E | N | N | N | N | N | N | N | ## Advanced SQL features -| Advanced SQL features | 7.4 | 7.3 | 7.2 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Prepared statement cache](/sql-prepared-plan-cache.md) | Y | Y | Y | Y | Y | Y | Y | Y | E | E | E | E | -| [Non-prepared statement cache](/sql-non-prepared-plan-cache.md) | Y | E | E | E | N | N | N | N | N | N | N | N | -| [SQL plan management (SPM)](/sql-plan-management.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Create bindings according to historical execution plans](/sql-plan-management.md#create-a-binding-according-to-a-historical-execution-plan) | Y | Y | Y | Y | E | N | N | N | N | N | N | N | -| [Coprocessor cache](/coprocessor-cache.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | E | -| [Stale Read](/stale-read.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | N | -| [Follower reads](/follower-read.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Read historical data (tidb_snapshot)](/read-historical-data.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Optimizer hints](/optimizer-hints.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [MPP execution engine](/explain-mpp.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | -| [MPP execution engine - compression exchange](/explain-mpp.md#mpp-version-and-exchange-data-compression) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [TiFlash Pipeline Model](/tiflash/tiflash-pipeline-model.md) | Y | E | E | N | N | N | N | N | N | N | N | N | -| [TiFlash replica selection strategy](/system-variables.md#tiflash_replica_read-new-in-v730) | Y | Y | N | N | N | N | N | N | N | N | N | N | -| [Index Merge](/explain-index-merge.md) | Y | Y | Y | Y | Y | Y | Y | E | E | E | E | E | -| [Placement Rules in SQL](/placement-rules-in-sql.md) | Y | Y | Y | Y | Y | Y | E | E | N | N | N | N | -| [Cascades Planner](/system-variables.md#tidb_enable_cascades_planner) | E | E | E | E | E | E | E | E | E | E | E | E | -| [Runtime Filter](/runtime-filter.md) | Y | Y | N | N | N | N | N | N | N | N | N | N | +| Advanced SQL features | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [Prepared statement cache](/sql-prepared-plan-cache.md) | Y | Y | Y | Y | Y | Y | E | E | E | E | +| [Non-prepared statement cache](/sql-non-prepared-plan-cache.md) | Y | E | N | N | N | N | N | N | N | N | +| [SQL plan management (SPM)](/sql-plan-management.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Create bindings according to historical execution plans](/sql-plan-management.md#create-a-binding-according-to-a-historical-execution-plan) | Y | Y | E | N | N | N | N | N | N | N | +| [Coprocessor cache](/coprocessor-cache.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | E | +| [Stale Read](/stale-read.md) | Y | Y | Y | Y | Y | Y | Y | Y | N | N | +| [Follower reads](/follower-read.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Read historical data (tidb_snapshot)](/read-historical-data.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Optimizer hints](/optimizer-hints.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [MPP execution engine](/explain-mpp.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | +| [MPP execution engine - compression exchange](/explain-mpp.md#mpp-version-and-exchange-data-compression) | Y | Y | N | N | N | N | N | N | N | N | +| [TiFlash Pipeline Model](/tiflash/tiflash-pipeline-model.md) | Y | N | N | N | N | N | N | N | N | N | +| [TiFlash replica selection strategy](/system-variables.md#tiflash_replica_read-new-in-v730) | Y | N | N | N | N | N | N | N | N | N | +| [Index Merge](/explain-index-merge.md) | Y | Y | Y | Y | Y | E | E | E | E | E | +| [Placement Rules in SQL](/placement-rules-in-sql.md) | Y | Y | Y | Y | E | E | N | N | N | N | +| [Cascades Planner](/system-variables.md#tidb_enable_cascades_planner) | E | E | E | E | E | E | E | E | E | E | +| [Runtime Filter](/runtime-filter.md) | Y | N | N | N | N | N | N | N | N | N | ## Data definition language (DDL) -| Data definition language (DDL) | 7.4 | 7.3 | 7.2 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| Basic `CREATE`, `DROP`, `ALTER`, `RENAME`, `TRUNCATE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Generated columns](/generated-columns.md) | Y | Y | Y | Y | E | E | E | E | E | E | E | E | -| [Views](/views.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Sequences](/sql-statements/sql-statement-create-sequence.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Auto increment](/auto-increment.md) | Y | Y | Y | Y | Y[^4] | Y | Y | Y | Y | Y | Y | Y | -| [Auto random](/auto-random.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [TTL (Time to Live)](/time-to-live.md) | Y | Y | Y | Y | E | N | N | N | N | N | N | N | -| [DDL algorithm assertions](/sql-statements/sql-statement-alter-table.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| Multi-schema change: add columns | Y | Y | Y | Y | Y | E | E | E | E | E | E | E | -| [Change column type](/sql-statements/sql-statement-modify-column.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | N | -| [Temporary tables](/temporary-tables.md) | Y | Y | Y | Y | Y | Y | Y | Y | N | N | N | N | -| Concurrent DDL statements | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [Acceleration of `ADD INDEX` and `CREATE INDEX`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [Metadata lock](/metadata-lock.md) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [`FLASHBACK CLUSTER TO TIMESTAMP`](/sql-statements/sql-statement-flashback-to-timestamp.md) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [Pause](/sql-statements/sql-statement-admin-pause-ddl.md)/[Resume](/sql-statements/sql-statement-admin-resume-ddl.md) DDL | E | E | E | N | N | N | N | N | N | N | N | N | +| Data definition language (DDL) | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| Basic `CREATE`, `DROP`, `ALTER`, `RENAME`, `TRUNCATE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Generated columns](/generated-columns.md) | Y | Y | E | E | E | E | E | E | E | E | +| [Views](/views.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Sequences](/sql-statements/sql-statement-create-sequence.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Auto increment](/auto-increment.md) | Y | Y | Y[^4] | Y | Y | Y | Y | Y | Y | Y | +| [Auto random](/auto-random.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [TTL (Time to Live)](/time-to-live.md) | Y | Y | E | N | N | N | N | N | N | N | +| [DDL algorithm assertions](/sql-statements/sql-statement-alter-table.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| Multi-schema change: add columns | Y | Y | Y | E | E | E | E | E | E | E | +| [Change column type](/sql-statements/sql-statement-modify-column.md) | Y | Y | Y | Y | Y | Y | Y | Y | N | N | +| [Temporary tables](/temporary-tables.md) | Y | Y | Y | Y | Y | Y | N | N | N | N | +| Concurrent DDL statements | Y | Y | Y | N | N | N | N | N | N | N | +| [Acceleration of `ADD INDEX` and `CREATE INDEX`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) | Y | Y | Y | N | N | N | N | N | N | N | +| [Metadata lock](/metadata-lock.md) | Y | Y | Y | N | N | N | N | N | N | N | +| [`FLASHBACK CLUSTER`](/sql-statements/sql-statement-flashback-cluster.md) | Y | Y | Y | N | N | N | N | N | N | N | +| [Pause](/sql-statements/sql-statement-admin-pause-ddl.md)/[Resume](/sql-statements/sql-statement-admin-resume-ddl.md) DDL | Y | N | N | N | N | N | N | N | N | N | ## Transactions -| Transactions | 7.4 | 7.3 | 7.2 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Async commit](/system-variables.md#tidb_enable_async_commit-new-in-v50) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | -| [1PC](/system-variables.md#tidb_enable_1pc-new-in-v50) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | -| [Large transactions (10GB)](/transaction-overview.md#transaction-size-limit) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Pessimistic transactions](/pessimistic-transaction.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Optimistic transactions](/optimistic-transaction.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Repeatable-read isolation (snapshot isolation)](/transaction-isolation-levels.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Read-committed isolation](/transaction-isolation-levels.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| Transactions | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [Async commit](/system-variables.md#tidb_enable_async_commit-new-in-v50) | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | +| [1PC](/system-variables.md#tidb_enable_1pc-new-in-v50) | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | +| [Large transactions (10GB)](/transaction-overview.md#transaction-size-limit) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Pessimistic transactions](/pessimistic-transaction.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Optimistic transactions](/optimistic-transaction.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Repeatable-read isolation (snapshot isolation)](/transaction-isolation-levels.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Read-committed isolation](/transaction-isolation-levels.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | ## Partitioning -| Partitioning | 7.4 | 7.3 | 7.2 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Range partitioning](/partitioned-table.md#range-partitioning) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Hash partitioning](/partitioned-table.md#hash-partitioning) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Key partitioning](/partitioned-table.md#key-partitioning) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [List partitioning](/partitioned-table.md#list-partitioning) | Y | Y | Y | Y | Y | Y | E | E | E | E | E | N | -| [List COLUMNS partitioning](/partitioned-table.md) | Y | Y | Y | Y | Y | Y | E | E | E | E | E | N | -| [Default partition for List and List COLUMNS partitioned tables](/partitioned-table.md#default-list-partition) | Y | Y | N | N | N | N | N | N | N | N | N | N | -| [`EXCHANGE PARTITION`](/partitioned-table.md) | Y | Y | Y | Y | Y | E | E | E | E | E | E | N | -| [`REORGANIZE PARTITION`](/partitioned-table.md#reorganize-partitions) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [`COALESCE PARTITION`](/partitioned-table.md#decrease-the-number-of-partitions) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [Dynamic pruning](/partitioned-table.md#dynamic-pruning-mode) | Y | Y | Y | Y | Y | Y | E | E | E | E | N | N | -| [Range COLUMNS partitioning](/partitioned-table.md#range-columns-partitioning) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [Range INTERVAL partitioning](/partitioned-table.md#range-interval-partitioning) | Y | Y | Y | Y | E | N | N | N | N | N | N | N | -| [Convert a partitioned table to a non-partitioned table](/partitioned-table.md#convert-a-partitioned-table-to-a-non-partitioned-table) | Y | N | N | N | N | N | N | N | N | N | N | N | -| [Partition an existing table](/partitioned-table.md#partition-an-existing-table) | Y | N | N | N | N | N | N | N | N | N | N | N | +| Partitioning | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [Range partitioning](/partitioned-table.md#range-partitioning) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Hash partitioning](/partitioned-table.md#hash-partitioning) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Key partitioning](/partitioned-table.md#key-partitioning) | Y | Y | N | N | N | N | N | N | N | N | +| [List partitioning](/partitioned-table.md#list-partitioning) | Y | Y | Y | Y | E | E | E | E | E | N | +| [List COLUMNS partitioning](/partitioned-table.md) | Y | Y | Y | Y | E | E | E | E | E | N | +| [Default partition for List and List COLUMNS partitioned tables](/partitioned-table.md#default-list-partition) | Y | N | N | N | N | N | N | N | N | N | +| [`EXCHANGE PARTITION`](/partitioned-table.md) | Y | Y | Y | E | E | E | E | E | E | N | +| [`REORGANIZE PARTITION`](/partitioned-table.md#reorganize-partitions) | Y | Y | N | N | N | N | N | N | N | N | +| [`COALESCE PARTITION`](/partitioned-table.md#decrease-the-number-of-partitions) | Y | Y | N | N | N | N | N | N | N | N | +| [Dynamic pruning](/partitioned-table.md#dynamic-pruning-mode) | Y | Y | Y | Y | E | E | E | E | N | N | +| [Range COLUMNS partitioning](/partitioned-table.md#range-columns-partitioning) | Y | Y | Y | N | N | N | N | N | N | N | +| [Range INTERVAL partitioning](/partitioned-table.md#range-interval-partitioning) | Y | Y | E | N | N | N | N | N | N | N | +| [Convert a partitioned table to a non-partitioned table](/partitioned-table.md#convert-a-partitioned-table-to-a-non-partitioned-table) | Y | N | N | N | N | N | N | N | N | N | +| [Partition an existing table](/partitioned-table.md#partition-an-existing-table) | Y | N | N | N | N | N | N | N | N | N | ## Statistics -| Statistics | 7.4 | 7.3 | 7.2 | 7.1 | 6.5 | 6.1 | 6.0 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [CMSketch](/statistics.md) | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Y | Y | Y | -| [Histograms](/statistics.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Extended statistics](/extended-statistics.md) | E | E | E | E | E | E | E | E | E | E | E | E | -| Statistics feedback | N | N | N | N | N | Deprecated | Deprecated | Deprecated | E | E | E | E | -| [Automatically update statistics](/statistics.md#automatic-update) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Fast Analyze](/system-variables.md#tidb_enable_fast_analyze) | E | E | E | E | E | E | E | E | E | E | E | E | -| [Dynamic pruning](/partitioned-table.md#dynamic-pruning-mode) | Y | Y | Y | Y | Y | Y | E | E | E | E | E | N | -| [Collect statistics for `PREDICATE COLUMNS`](/statistics.md#collect-statistics-on-some-columns) | E | E | E | E | E | E | E | E | N | N | N | N | -| [Control the memory quota for collecting statistics](/statistics.md#the-memory-quota-for-collecting-statistics) | E | E | E | E | E | E | N | N | N | N | N | N | -| [Randomly sample about 10000 rows of data to quickly build statistics](/system-variables.md#tidb_enable_fast_analyze) | E | E | E | E | E | E | E | E | E | E | E | E | -| [Lock statistics](/statistics.md#lock-statistics) | E | E | E | E | E | N | N | N | N | N | N | N | -| [Lightweight statistics initialization](/statistics.md#load-statistics) | Y | Y | Y | E | N | N | N | N | N | N | N | N | -| [Show the progress of collecting statistics](/sql-statements/sql-statement-show-analyze-status.md) | Y | Y | N | N | N | N | N | N | N | N | N | N | +| Statistics | 7.5 | 7.1 | 6.5 | 6.1 | 6.0 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [CMSketch](/statistics.md) | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Y | Y | Y | +| [Histograms](/statistics.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Extended statistics](/extended-statistics.md) | E | E | E | E | E | E | E | E | E | E | +| Statistics feedback | N | N | N | Deprecated | Deprecated | Deprecated | E | E | E | E | +| [Automatically update statistics](/statistics.md#automatic-update) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Dynamic pruning](/partitioned-table.md#dynamic-pruning-mode) | Y | Y | Y | Y | E | E | E | E | E | N | +| [Collect statistics for `PREDICATE COLUMNS`](/statistics.md#collect-statistics-on-some-columns) | E | E | E | E | E | E | N | N | N | N | +| [Control the memory quota for collecting statistics](/statistics.md#the-memory-quota-for-collecting-statistics) | E | E | E | E | N | N | N | N | N | N | +| [Randomly sample about 10000 rows of data to quickly build statistics](/system-variables.md#tidb_enable_fast_analyze) | Deprecated | E | E | E | E | E | E | E | E | E | +| [Lock statistics](/statistics.md#lock-statistics) | Y | E | E | N | N | N | N | N | N | N | +| [Lightweight statistics initialization](/statistics.md#load-statistics) | Y | E | N | N | N | N | N | N | N | N | +| [Show the progress of collecting statistics](/sql-statements/sql-statement-show-analyze-status.md) | Y | N | N | N | N | N | N | N | N | N | ## Security -| Security | 7.4 | 7.3 | 7.2 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Transparent layer security (TLS)](/enable-tls-between-clients-and-servers.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Encryption at rest (TDE)](/encryption-at-rest.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Role-based authentication (RBAC)](/role-based-access-control.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Certificate-based authentication](/certificate-authentication.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [`caching_sha2_password` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | N | N | -| [`tidb_sm3_password` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [`tidb_auth_token` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [`authentication_ldap_sasl` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | N | N | N | N | N | N | N | N | -| [`authentication_ldap_simple` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [Password management](/password-management.md) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [MySQL compatible `GRANT` system](/privilege-management.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Dynamic Privileges](/privilege-management.md#dynamic-privileges) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | N | -| [Security Enhanced Mode](/system-variables.md#tidb_enable_enhanced_security) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | N | -| [Redacted Log Files](/log-redaction.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | +| Security | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [Transparent layer security (TLS)](/enable-tls-between-clients-and-servers.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Encryption at rest (TDE)](/encryption-at-rest.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Role-based authentication (RBAC)](/role-based-access-control.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Certificate-based authentication](/certificate-authentication.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [`caching_sha2_password` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | Y | Y | Y | Y | N | N | N | +| [`tidb_sm3_password` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | N | N | N | N | N | N | N | +| [`tidb_auth_token` authentication](/security-compatibility-with-mysql.md#tidb_auth_token) | Y | Y | Y | N | N | N | N | N | N | N | +| [`authentication_ldap_sasl` authentication](/system-variables.md#default_authentication_plugin) | Y | N | N | N | N | N | N | N | N | +| [`authentication_ldap_simple` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | N | N | N | N | N | N | N | N | +| [Password management](/password-management.md) | Y | Y | Y | N | N | N | N | N | N | N | +| [MySQL compatible `GRANT` system](/privilege-management.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Dynamic Privileges](/privilege-management.md#dynamic-privileges) | Y | Y | Y | Y | Y | Y | Y | Y | N | N | +| [Security Enhanced Mode](/system-variables.md#tidb_enable_enhanced_security) | Y | Y | Y | Y | Y | Y | Y | Y | N | N | +| [Redacted Log Files](/log-redaction.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | ## Data import and export -| Data import and export | 7.4 | 7.3 | 7.2 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Fast import using TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Fast import using the `IMPORT INTO` statement](/sql-statements/sql-statement-import-into.md) | E | E | E | N | N | N | N | N | N | N | N | N | -| mydumper logical dumper | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | -| [Dumpling logical dumper](/dumpling-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Transactional `LOAD DATA`](/sql-statements/sql-statement-load-data.md) [^5] | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N [^6] | -| [Database migration toolkit (DM)](/migration-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [TiDB Binlog](/tidb-binlog/tidb-binlog-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Change data capture (CDC)](/ticdc/ticdc-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Stream data to Amazon S3, GCS, Azure Blob Storage, and NFS through TiCDC](/ticdc/ticdc-sink-to-cloud-storage.md) | Y | Y | Y | Y | E | N | N | N | N | N | N | N | -| [TiCDC supports bidirectional replication between two TiDB clusters](/ticdc/ticdc-bidirectional-replication.md) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [TiCDC OpenAPI v2](/ticdc/ticdc-open-api-v2.md) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | +| Data import and export | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [Fast import using TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Fast import using the `IMPORT INTO` statement](/sql-statements/sql-statement-import-into.md) | Y | N | N | N | N | N | N | N | N | N | +| mydumper logical dumper | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | +| [Dumpling logical dumper](/dumpling-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Transactional `LOAD DATA`](/sql-statements/sql-statement-load-data.md) [^5] | Y | Y | Y | Y | Y | Y | Y | Y | Y | N [^6] | +| [Database migration toolkit (DM)](/migration-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [TiDB Binlog](/tidb-binlog/tidb-binlog-overview.md) [^7] | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Change data capture (CDC)](/ticdc/ticdc-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Stream data to Amazon S3, GCS, Azure Blob Storage, and NFS through TiCDC](/ticdc/ticdc-sink-to-cloud-storage.md) | Y | Y | E | N | N | N | N | N | N | N | +| [TiCDC supports bidirectional replication between two TiDB clusters](/ticdc/ticdc-bidirectional-replication.md) | Y | Y | Y | N | N | N | N | N | N | N | +| [TiCDC OpenAPI v2](/ticdc/ticdc-open-api-v2.md) | Y | Y | N | N | N | N | N | N | N | N | ## Management, observability, and tools -| Management, observability, and tools | 7.4 | 7.3 | 7.2 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [TiDB Dashboard UI](/dashboard/dashboard-intro.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [TiDB Dashboard Continuous Profiling](/dashboard/continuous-profiling.md) | Y | Y | Y | Y | Y | Y | E | E | N | N | N | N | -| [TiDB Dashboard Top SQL](/dashboard/top-sql.md) | Y | Y | Y | Y | Y | Y | E | N | N | N | N | N | -| [TiDB Dashboard SQL Diagnostics](/information-schema/information-schema-sql-diagnostics.md) | Y | Y | Y | Y | Y | E | E | E | E | E | E | E | -| [TiDB Dashboard Cluster Diagnostics](/dashboard/dashboard-diagnostics-access.md) | Y | Y | Y | Y | Y | E | E | E | E | E | E | E | -| [TiKV-FastTune dashboard](/grafana-tikv-dashboard.md#tikv-fasttune-dashboard) | E | E | E | E | E | E | E | E | E | E | E | E | -| [Information schema](/information-schema/information-schema.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Metrics schema](/metrics-schema.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Statements summary tables](/statement-summary-tables.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Statements summary tables - summary persistence](/statement-summary-tables.md#persist-statements-summary) | E | E | E | E | N | N | N | N | N | N | N | N | -| [Slow query log](/identify-slow-queries.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [TiUP deployment](/tiup/tiup-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Kubernetes operator](https://docs.pingcap.com/tidb-in-kubernetes/) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Built-in physical backup](/br/backup-and-restore-use-cases.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Global Kill](/sql-statements/sql-statement-kill.md) | Y | Y | Y | Y | Y | Y | E | E | E | E | E | E | -| [Lock View](/information-schema/information-schema-data-lock-waits.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | E | E | E | -| [`SHOW CONFIG`](/sql-statements/sql-statement-show-config.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [`SET CONFIG`](/dynamic-config.md) | Y | Y | Y | Y | Y | Y | E | E | E | E | E | E | -| [DM WebUI](/dm/dm-webui-guide.md) | E | E | E | E | E | E | N | N | N | N | N | N | -| [Foreground Quota Limiter](/tikv-configuration-file.md#foreground-quota-limiter) | Y | Y | Y | Y | Y | E | N | N | N | N | N | N | -| [Background Quota Limiter](/tikv-configuration-file.md#background-quota-limiter) | E | E | E | E | E | N | N | N | N | N | N | N | -| [EBS volume snapshot backup and restore](https://docs.pingcap.com/tidb-in-kubernetes/v1.4/backup-to-aws-s3-by-snapshot) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [PITR](/br/backup-and-restore-overview.md) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [Global memory control](/configure-memory-usage.md#configure-the-memory-usage-threshold-of-a-tidb-server-instance) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [Cross-cluster RawKV replication](/tikv-configuration-file.md#api-version-new-in-v610) | E | E | E | E | E | N | N | N | N | N | N | N | -| [Green GC](/system-variables.md#tidb_gc_scan_lock_mode-new-in-v50) | E | E | E | E | E | E | E | E | E | E | E | N | -| [Resource control](/tidb-resource-control.md) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [Runaway Queries management](/tidb-resource-control.md#manage-queries-that-consume-more-resources-than-expected-runaway-queries) | E | E | E | N | N | N | N | N | N | N | N | N | -| [Background tasks management](/tidb-resource-control.md#manage-background-tasks) | E | N | N | N | N | N | N | N | N | N | N | N | -| [TiFlash Disaggregated Storage and Compute Architecture and S3 Support](/tiflash/tiflash-disaggregated-and-s3.md) | Y | E | E | E | N | N | N | N | N | N | N | N | -| [Selecting TiDB nodes for the distributed framework tasks](/system-variables.md#tidb_service_scope-new-in-v740) | E | N | N | N | N | N | N | N | N | N | N | N | +| Management, observability, and tools | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [TiDB Dashboard UI](/dashboard/dashboard-intro.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [TiDB Dashboard Continuous Profiling](/dashboard/continuous-profiling.md) | Y | Y | Y | Y | E | E | N | N | N | N | +| [TiDB Dashboard Top SQL](/dashboard/top-sql.md) | Y | Y | Y | Y | E | N | N | N | N | N | +| [TiDB Dashboard SQL Diagnostics](/information-schema/information-schema-sql-diagnostics.md) | Y | Y | Y | E | E | E | E | E | E | E | +| [TiDB Dashboard Cluster Diagnostics](/dashboard/dashboard-diagnostics-access.md) | Y | Y | Y | E | E | E | E | E | E | E | +| [TiKV-FastTune dashboard](/grafana-tikv-dashboard.md#tikv-fasttune-dashboard) | E | E | E | E | E | E | E | E | E | E | +| [Information schema](/information-schema/information-schema.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Metrics schema](/metrics-schema.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Statements summary tables](/statement-summary-tables.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Statements summary tables - summary persistence](/statement-summary-tables.md#persist-statements-summary) | E | E | N | N | N | N | N | N | N | N | +| [Slow query log](/identify-slow-queries.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [TiUP deployment](/tiup/tiup-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Kubernetes operator](https://docs.pingcap.com/tidb-in-kubernetes/) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Built-in physical backup](/br/backup-and-restore-use-cases.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Global Kill](/sql-statements/sql-statement-kill.md) | Y | Y | Y | Y | E | E | E | E | E | E | +| [Lock View](/information-schema/information-schema-data-lock-waits.md) | Y | Y | Y | Y | Y | Y | Y | E | E | E | +| [`SHOW CONFIG`](/sql-statements/sql-statement-show-config.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [`SET CONFIG`](/dynamic-config.md) | Y | Y | Y | Y | E | E | E | E | E | E | +| [DM WebUI](/dm/dm-webui-guide.md) | E | E | E | E | N | N | N | N | N | N | +| [Foreground Quota Limiter](/tikv-configuration-file.md#foreground-quota-limiter) | Y | Y | Y | E | N | N | N | N | N | N | +| [Background Quota Limiter](/tikv-configuration-file.md#background-quota-limiter) | E | E | E | N | N | N | N | N | N | N | +| [EBS volume snapshot backup and restore](https://docs.pingcap.com/tidb-in-kubernetes/v1.4/backup-to-aws-s3-by-snapshot) | Y | Y | Y | N | N | N | N | N | N | N | +| [PITR](/br/backup-and-restore-overview.md) | Y | Y | Y | N | N | N | N | N | N | N | +| [Global memory control](/configure-memory-usage.md#configure-the-memory-usage-threshold-of-a-tidb-server-instance) | Y | Y | Y | N | N | N | N | N | N | N | +| [Cross-cluster RawKV replication](/tikv-configuration-file.md#api-version-new-in-v610) | E | E | E | N | N | N | N | N | N | N | +| [Green GC](/system-variables.md#tidb_gc_scan_lock_mode-new-in-v50) | E | E | E | E | E | E | E | E | E | N | +| [Resource control](/tidb-resource-control.md) | Y | Y | N | N | N | N | N | N | N | N | +| [Runaway Queries management](/tidb-resource-control.md#manage-queries-that-consume-more-resources-than-expected-runaway-queries) | E | N | N | N | N | N | N | N | N | N | +| [Background tasks management](/tidb-resource-control.md#manage-background-tasks) | E | N | N | N | N | N | N | N | N | N | +| [TiFlash Disaggregated Storage and Compute Architecture and S3 Support](/tiflash/tiflash-disaggregated-and-s3.md) | Y | E | N | N | N | N | N | N | N | N | +| [Selecting TiDB nodes for the Distributed eXecution Framework (DXF) tasks](/system-variables.md#tidb_service_scope-new-in-v740) | Y | N | N | N | N | N | N | N | N | N | [^1]: TiDB incorrectly treats latin1 as a subset of utf8. See [TiDB #18955](https://github.com/pingcap/tidb/issues/18955) for more details. @@ -263,3 +261,5 @@ You can try out TiDB features on [TiDB Playground](https://play.tidbcloud.com/?u [^5]: Starting from [TiDB v7.0.0](/releases/release-7.0.0.md), the new parameter `FIELDS DEFINED NULL BY` and support for importing data from S3 and GCS are experimental features. [^6]: For TiDB v4.0, the `LOAD DATA` transaction does not guarantee atomicity. + +[^7]: Starting from TiDB v7.5.0, technical support for the data replication feature of [TiDB Binlog](/tidb-binlog/tidb-binlog-overview.md) is no longer provided. It is strongly recommended to use [TiCDC](/ticdc/ticdc-overview.md) as an alternative solution for data replication. Although TiDB Binlog v7.5.0 still supports the Point-in-Time Recovery (PITR) scenario, this component will be completely deprecated in future versions. It is recommended to use [PITR](/br/br-pitr-guide.md) as an alternative solution for data recovery. diff --git a/basic-sql-operations.md b/basic-sql-operations.md index 6bd04f4324776..bd6c2eb43e3e3 100644 --- a/basic-sql-operations.md +++ b/basic-sql-operations.md @@ -1,7 +1,6 @@ --- title: Explore SQL with TiDB summary: Learn about the basic SQL statements for the TiDB database. -aliases: ['/docs/dev/basic-sql-operations/','/docs/dev/how-to/get-started/explore-sql/'] --- # Explore SQL with TiDB diff --git a/benchmark/benchmark-sysbench-v2.md b/benchmark/benchmark-sysbench-v2.md index 24a29886d24f4..93168caab39fc 100644 --- a/benchmark/benchmark-sysbench-v2.md +++ b/benchmark/benchmark-sysbench-v2.md @@ -1,6 +1,6 @@ --- title: TiDB Sysbench Performance Test Report -- v2.0.0 vs. v1.0.0 -aliases: ['/docs/dev/benchmark/benchmark-sysbench-v2/','/docs/dev/benchmark/sysbench-v2/'] +summary: TiDB 2.0 GA outperforms TiDB 1.0 GA in `Select` and `Insert` tests, with a 10% increase in `Select` query performance and a slight improvement in `Insert` query performance. However, the OLTP performance of both versions is almost the same. --- # TiDB Sysbench Performance Test Report -- v2.0.0 vs. v1.0.0 diff --git a/benchmark/benchmark-sysbench-v3.md b/benchmark/benchmark-sysbench-v3.md index 91308e70ee734..9aa795659eaec 100644 --- a/benchmark/benchmark-sysbench-v3.md +++ b/benchmark/benchmark-sysbench-v3.md @@ -1,6 +1,6 @@ --- title: TiDB Sysbench Performance Test Report -- v2.1 vs. v2.0 -aliases: ['/docs/dev/benchmark/benchmark-sysbench-v3/','/docs/dev/benchmark/sysbench-v3/'] +summary: TiDB 2.1 outperforms TiDB 2.0 in the `Point Select` test, with a 50% increase in query performance. However, the `Update Non-Index` and `Update Index` tests show similar performance between the two versions. The test was conducted in September 2018 in Beijing, China, using a specific test environment and configuration. --- # TiDB Sysbench Performance Test Report -- v2.1 vs. v2.0 diff --git a/benchmark/benchmark-sysbench-v4-vs-v3.md b/benchmark/benchmark-sysbench-v4-vs-v3.md index 41fd3a544de22..1c997eb2b8316 100644 --- a/benchmark/benchmark-sysbench-v4-vs-v3.md +++ b/benchmark/benchmark-sysbench-v4-vs-v3.md @@ -1,7 +1,6 @@ --- title: TiDB Sysbench Performance Test Report -- v4.0 vs. v3.0 summary: Compare the Sysbench performance of TiDB 4.0 and TiDB 3.0. -aliases: ['/docs/dev/benchmark/benchmark-sysbench-v4-vs-v3/'] --- # TiDB Sysbench Performance Test Report -- v4.0 vs. v3.0 diff --git a/benchmark/benchmark-sysbench-v5-vs-v4.md b/benchmark/benchmark-sysbench-v5-vs-v4.md index 871f05eb90c90..c214c0e2955a9 100644 --- a/benchmark/benchmark-sysbench-v5-vs-v4.md +++ b/benchmark/benchmark-sysbench-v5-vs-v4.md @@ -1,5 +1,6 @@ --- title: TiDB Sysbench Performance Test Report -- v5.0 vs. v4.0 +summary: TiDB v5.0 outperforms v4.0 in Sysbench performance tests. Point Select performance improved by 2.7%, Update Non-index by 81%, Update Index by 28%, and Read Write by 9%. The test aimed to compare performance in the OLTP scenario using AWS EC2. Test results were presented in tables and graphs. --- # TiDB Sysbench Performance Test Report -- v5.0 vs. v4.0 @@ -108,7 +109,7 @@ set global tidb_hashagg_partial_concurrency=1; set global tidb_enable_async_commit = 1; set global tidb_enable_1pc = 1; set global tidb_guarantee_linearizability = 0; -set global tidb_enable_clustered_index = 1; +set global tidb_enable_clustered_index = 1; ``` diff --git a/benchmark/benchmark-sysbench-v5.1.0-vs-v5.0.2.md b/benchmark/benchmark-sysbench-v5.1.0-vs-v5.0.2.md index 5e413d27b2b33..a9d398939913f 100644 --- a/benchmark/benchmark-sysbench-v5.1.0-vs-v5.0.2.md +++ b/benchmark/benchmark-sysbench-v5.1.0-vs-v5.0.2.md @@ -1,5 +1,6 @@ --- title: TiDB Sysbench Performance Test Report -- v5.1.0 vs. v5.0.2 +summary: TiDB v5.1.0 shows a 19.4% improvement in Point Select performance compared to v5.0.2. However, the Read Write and Update Index performance is slightly reduced in v5.1.0. The test was conducted on AWS EC2 using Sysbench with specific hardware and software configurations. The test plan involved deploying, importing data, and performing stress tests. Overall, v5.1.0 demonstrates improved Point Select performance but reduced performance in other areas. --- # TiDB Sysbench Performance Test Report -- v5.1.0 vs. v5.0.2 diff --git a/benchmark/benchmark-sysbench-v5.2.0-vs-v5.1.1.md b/benchmark/benchmark-sysbench-v5.2.0-vs-v5.1.1.md index 87064ac5f3234..ae539a95e551b 100644 --- a/benchmark/benchmark-sysbench-v5.2.0-vs-v5.1.1.md +++ b/benchmark/benchmark-sysbench-v5.2.0-vs-v5.1.1.md @@ -1,5 +1,6 @@ --- title: TiDB Sysbench Performance Test Report -- v5.2.0 vs. v5.1.1 +summary: TiDB v5.2.0 shows an 11.03% improvement in Point Select performance compared to v5.1.1. However, other scenarios show a slight reduction in performance. The hardware and software configurations, test plan, and results are detailed in the report. --- # TiDB Sysbench Performance Test Report -- v5.2.0 vs. v5.1.1 diff --git a/benchmark/benchmark-sysbench-v5.3.0-vs-v5.2.2.md b/benchmark/benchmark-sysbench-v5.3.0-vs-v5.2.2.md index 2a5e44374dcdc..8759d0928e53e 100644 --- a/benchmark/benchmark-sysbench-v5.3.0-vs-v5.2.2.md +++ b/benchmark/benchmark-sysbench-v5.3.0-vs-v5.2.2.md @@ -1,5 +1,6 @@ --- title: TiDB Sysbench Performance Test Report -- v5.3.0 vs. v5.2.2 +summary: TiDB v5.3.0 and v5.2.2 were compared in a Sysbench performance test for Online Transactional Processing (OLTP). Results show that v5.3.0 performance is nearly the same as v5.2.2. Point Select performance of v5.3.0 is reduced by 0.81%, Update Non-index performance is improved by 0.95%, Update Index performance is improved by 1.83%, and Read Write performance is reduced by 0.62%. --- # TiDB Sysbench Performance Test Report -- v5.3.0 vs. v5.2.2 diff --git a/benchmark/benchmark-sysbench-v5.4.0-vs-v5.3.0.md b/benchmark/benchmark-sysbench-v5.4.0-vs-v5.3.0.md index 476e056ea38eb..c1b0c055e805f 100644 --- a/benchmark/benchmark-sysbench-v5.4.0-vs-v5.3.0.md +++ b/benchmark/benchmark-sysbench-v5.4.0-vs-v5.3.0.md @@ -1,5 +1,6 @@ --- title: TiDB Sysbench Performance Test Report -- v5.4.0 vs. v5.3.0 +summary: TiDB v5.4.0 shows improved performance of 2.59% to 4.85% in write-heavy workloads compared to v5.3.0. Results show performance improvements in point select, update non-index, update index, and read write scenarios. --- # TiDB Sysbench Performance Test Report -- v5.4.0 vs. v5.3.0 diff --git a/benchmark/benchmark-sysbench-v6.0.0-vs-v5.4.0.md b/benchmark/benchmark-sysbench-v6.0.0-vs-v5.4.0.md index 2948929d23136..0af44a97299aa 100644 --- a/benchmark/benchmark-sysbench-v6.0.0-vs-v5.4.0.md +++ b/benchmark/benchmark-sysbench-v6.0.0-vs-v5.4.0.md @@ -1,5 +1,6 @@ --- title: TiDB Sysbench Performance Test Report -- v6.0.0 vs. v5.4.0 +summary: TiDB v6.0.0 shows a 16.17% improvement in read-write workload performance compared to v5.4.0. Other workloads show similar performance between the two versions. Test results show performance comparisons for point select, update non-index, update index, and read-write workloads. --- # TiDB Sysbench Performance Test Report -- v6.0.0 vs. v5.4.0 diff --git a/benchmark/benchmark-sysbench-v6.1.0-vs-v6.0.0.md b/benchmark/benchmark-sysbench-v6.1.0-vs-v6.0.0.md index 12ae8bf100d81..a61d18baf57ea 100644 --- a/benchmark/benchmark-sysbench-v6.1.0-vs-v6.0.0.md +++ b/benchmark/benchmark-sysbench-v6.1.0-vs-v6.0.0.md @@ -1,5 +1,6 @@ --- title: TiDB Sysbench Performance Test Report -- v6.1.0 vs. v6.0.0 +summary: TiDB v6.1.0 shows improved performance in write-heavy workloads compared to v6.0.0, with a 2.33% ~ 4.61% improvement. The test environment includes AWS EC2 instances and Sysbench 1.1.0-df89d34. Both versions use the same parameter configuration. Test plan involves deploying, importing data, and performing stress tests. Results show slight drop in Point Select performance, while Update Non-index, Update Index, and Read Write performance are improved by 2.90%, 4.61%, and 2.23% respectively. --- # TiDB Sysbench Performance Test Report -- v6.1.0 vs. v6.0.0 diff --git a/benchmark/benchmark-sysbench-v6.2.0-vs-v6.1.0.md b/benchmark/benchmark-sysbench-v6.2.0-vs-v6.1.0.md index 22999f9c2886d..f98bbf3130b13 100644 --- a/benchmark/benchmark-sysbench-v6.2.0-vs-v6.1.0.md +++ b/benchmark/benchmark-sysbench-v6.2.0-vs-v6.1.0.md @@ -1,5 +1,6 @@ --- title: TiDB Sysbench Performance Test Report -- v6.2.0 vs. v6.1.0 +summary: TiDB v6.2.0 and v6.1.0 show similar performance in the Sysbench test. Point Select performance slightly drops by 3.58%. Update Non-index and Update Index performance are basically unchanged, reduced by 0.85% and 0.47% respectively. Read Write performance is reduced by 1.21%. --- # TiDB Sysbench Performance Test Report -- v6.2.0 vs. v6.1.0 diff --git a/benchmark/benchmark-tidb-using-sysbench.md b/benchmark/benchmark-tidb-using-sysbench.md index 35b8ddd0758a7..38dca1f519f68 100644 --- a/benchmark/benchmark-tidb-using-sysbench.md +++ b/benchmark/benchmark-tidb-using-sysbench.md @@ -1,6 +1,6 @@ --- title: How to Test TiDB Using Sysbench -aliases: ['/docs/dev/benchmark/benchmark-tidb-using-sysbench/','/docs/dev/benchmark/how-to-run-sysbench/'] +summary: TiDB performance can be optimized by using Sysbench 1.0 or later. Configure TiDB and TiKV with higher log levels for better performance. Adjust Sysbench configuration and import data to optimize performance. Address common issues related to proxy use and CPU utilization rates. --- # How to Test TiDB Using Sysbench @@ -19,7 +19,7 @@ server_configs: log.level: "error" ``` -It is also recommended to make sure [`tidb_enable_prepared_plan_cache`](/system-variables.md#tidb_enable_prepared_plan_cache-new-in-v610) is enabled and that you allow sysbench to use prepared statements by using `--db-ps-mode=auto`. See the [SQL Prepared Execution Plan Cache](/sql-prepared-plan-cache.md) for documetnation about what the SQL plan cache does and how to monitor it. +It is also recommended to make sure [`tidb_enable_prepared_plan_cache`](/system-variables.md#tidb_enable_prepared_plan_cache-new-in-v610) is enabled and that you allow sysbench to use prepared statements by using `--db-ps-mode=auto`. See the [SQL Prepared Execution Plan Cache](/sql-prepared-plan-cache.md) for documentation about what the SQL plan cache does and how to monitor it. > **Note:** > diff --git a/benchmark/benchmark-tidb-using-tpcc.md b/benchmark/benchmark-tidb-using-tpcc.md index 170f36e68a5f7..d59867c3b6476 100644 --- a/benchmark/benchmark-tidb-using-tpcc.md +++ b/benchmark/benchmark-tidb-using-tpcc.md @@ -1,6 +1,6 @@ --- title: How to Run TPC-C Test on TiDB -aliases: ['/docs/dev/benchmark/benchmark-tidb-using-tpcc/','/docs/dev/benchmark/how-to-run-tpcc/'] +summary: This document describes how to test TiDB using TPC-C, an online transaction processing benchmark. It specifies the initial state of the database, provides commands for loading data, running the test, and cleaning up test data. The test measures the maximum qualified throughput using tpmC (transactions per minute). --- # How to Run TPC-C Test on TiDB diff --git a/benchmark/benchmark-tpch.md b/benchmark/benchmark-tpch.md index ee2b48fccd9ff..20c740294d503 100644 --- a/benchmark/benchmark-tpch.md +++ b/benchmark/benchmark-tpch.md @@ -1,6 +1,6 @@ --- title: TiDB TPC-H 50G Performance Test Report V2.0 -aliases: ['/docs/dev/benchmark/benchmark-tpch/','/docs/dev/benchmark/tpch/'] +summary: TiDB TPC-H 50G Performance Test compared TiDB 1.0 and TiDB 2.0 in an OLAP scenario. Test results show that TiDB 2.0 outperformed TiDB 1.0 in most queries, with significant improvements in query processing time. Some queries in TiDB 1.0 did not return results, while others had high memory consumption. Future releases plan to support VIEW and address these issues. --- # TiDB TPC-H 50G Performance Test Report diff --git a/benchmark/online-workloads-and-add-index-operations.md b/benchmark/online-workloads-and-add-index-operations.md index 366cf3adb3fb7..dafc6462c39c6 100644 --- a/benchmark/online-workloads-and-add-index-operations.md +++ b/benchmark/online-workloads-and-add-index-operations.md @@ -1,7 +1,6 @@ --- title: Interaction Test on Online Workloads and `ADD INDEX` Operations summary: This document tests the interaction effects between online workloads and `ADD INDEX` operations. -aliases: ['/docs/dev/benchmark/online-workloads-and-add-index-operations/','/docs/dev/benchmark/add-index-with-load/'] --- # Interaction Test on Online Workloads and `ADD INDEX` Operations diff --git a/benchmark/v3.0-performance-benchmarking-with-sysbench.md b/benchmark/v3.0-performance-benchmarking-with-sysbench.md index 03b8a413dff87..085203dc773d9 100644 --- a/benchmark/v3.0-performance-benchmarking-with-sysbench.md +++ b/benchmark/v3.0-performance-benchmarking-with-sysbench.md @@ -1,6 +1,6 @@ --- title: TiDB Sysbench Performance Test Report -- v3.0 vs. v2.1 -aliases: ['/docs/dev/benchmark/v3.0-performance-benchmarking-with-sysbench/','/docs/dev/benchmark/sysbench-v4/'] +summary: TiDB v3.0 outperformed v2.1 in all tests, with higher QPS and lower latency. Configuration changes in v3.0 contributed to the improved performance. --- # TiDB Sysbench Performance Test Report -- v3.0 vs. v2.1 diff --git a/benchmark/v3.0-performance-benchmarking-with-tpcc.md b/benchmark/v3.0-performance-benchmarking-with-tpcc.md index 920f8efa511e5..0d902d207d405 100644 --- a/benchmark/v3.0-performance-benchmarking-with-tpcc.md +++ b/benchmark/v3.0-performance-benchmarking-with-tpcc.md @@ -1,6 +1,6 @@ --- title: TiDB TPC-C Performance Test Report -- v3.0 vs. v2.1 -aliases: ['/docs/dev/benchmark/v3.0-performance-benchmarking-with-tpcc/','/docs/dev/benchmark/tpcc/'] +summary: TiDB v3.0 outperforms v2.1 in TPC-C performance test. With 1000 warehouses, v3.0 achieved 450% higher performance than v2.1. --- # TiDB TPC-C Performance Test Report -- v3.0 vs. v2.1 diff --git a/benchmark/v4.0-performance-benchmarking-with-tpcc.md b/benchmark/v4.0-performance-benchmarking-with-tpcc.md index 1169a9a92f492..bc7ad745b8824 100644 --- a/benchmark/v4.0-performance-benchmarking-with-tpcc.md +++ b/benchmark/v4.0-performance-benchmarking-with-tpcc.md @@ -1,7 +1,6 @@ --- title: TiDB TPC-C Performance Test Report -- v4.0 vs. v3.0 summary: Compare the TPC-C performance of TiDB 4.0 and TiDB 3.0 using BenchmarkSQL. -aliases: ['/docs/dev/benchmark/v4.0-performance-benchmarking-with-tpcc/'] --- # TiDB TPC-C Performance Test Report -- v4.0 vs. v3.0 diff --git a/benchmark/v4.0-performance-benchmarking-with-tpch.md b/benchmark/v4.0-performance-benchmarking-with-tpch.md index 801db52f05b69..e7364e64b85e7 100644 --- a/benchmark/v4.0-performance-benchmarking-with-tpch.md +++ b/benchmark/v4.0-performance-benchmarking-with-tpch.md @@ -1,7 +1,6 @@ --- title: TiDB TPC-H Performance Test Report -- v4.0 vs. v3.0 summary: Compare the TPC-H performance of TiDB 4.0 and TiDB 3.0. -aliases: ['/docs/dev/benchmark/v4.0-performance-benchmarking-with-tpch/'] --- # TiDB TPC-H Performance Test Report -- v4.0 vs. v3.0 diff --git a/benchmark/v5.0-performance-benchmarking-with-tpcc.md b/benchmark/v5.0-performance-benchmarking-with-tpcc.md index cc3e969fba028..5472e67664cca 100644 --- a/benchmark/v5.0-performance-benchmarking-with-tpcc.md +++ b/benchmark/v5.0-performance-benchmarking-with-tpcc.md @@ -1,5 +1,6 @@ --- title: TiDB TPC-C Performance Test Report -- v5.0 vs. v4.0 +summary: TiDB v5.0 outperforms v4.0 in TPC-C performance, showing a 36% increase. --- # TiDB TPC-C Performance Test Report -- v5.0 vs. v4.0 diff --git a/benchmark/v5.1-performance-benchmarking-with-tpcc.md b/benchmark/v5.1-performance-benchmarking-with-tpcc.md index dcc9e4c29ba91..a9d8a26503b69 100644 --- a/benchmark/v5.1-performance-benchmarking-with-tpcc.md +++ b/benchmark/v5.1-performance-benchmarking-with-tpcc.md @@ -1,5 +1,6 @@ --- title: TiDB TPC-C Performance Test Report -- v5.1.0 vs. v5.0.2 +summary: TiDB v5.1.0 TPC-C performance is 2.8% better than v5.0.2. Parameter configuration is the same for both versions. Test plan includes deployment, database creation, data import, stress testing, and result extraction. --- # TiDB TPC-C Performance Test Report -- v5.1.0 vs. v5.0.2 diff --git a/benchmark/v5.2-performance-benchmarking-with-tpcc.md b/benchmark/v5.2-performance-benchmarking-with-tpcc.md index 42c98cabf145a..8d26565e7f607 100644 --- a/benchmark/v5.2-performance-benchmarking-with-tpcc.md +++ b/benchmark/v5.2-performance-benchmarking-with-tpcc.md @@ -1,5 +1,6 @@ --- title: TiDB TPC-C Performance Test Report -- v5.2.0 vs. v5.1.1 +summary: TiDB v5.2.0 TPC-C performance is 4.22% lower than v5.1.1. Test environment AWS EC2. Hardware and software configurations are the same for both versions. Test plan includes deployment, database creation, data import, stress testing, and result extraction. --- # TiDB TPC-C Performance Test Report -- v5.2.0 vs. v5.1.1 diff --git a/benchmark/v5.3-performance-benchmarking-with-tpcc.md b/benchmark/v5.3-performance-benchmarking-with-tpcc.md index 7bc6139810e72..a5a4c35261850 100644 --- a/benchmark/v5.3-performance-benchmarking-with-tpcc.md +++ b/benchmark/v5.3-performance-benchmarking-with-tpcc.md @@ -1,5 +1,6 @@ --- title: TiDB TPC-C Performance Test Report -- v5.3.0 vs. v5.2.2 +summary: TiDB v5.3.0 TPC-C performance is slightly reduced by 2.99% compared to v5.2.2. The test used AWS EC2 with specific hardware and software configurations. The test plan involved deploying TiDB, creating a database, importing data, and running stress tests. The result showed a decrease in performance across different thread counts. --- # TiDB TPC-C Performance Test Report -- v5.3.0 vs. v5.2.2 diff --git a/benchmark/v5.4-performance-benchmarking-with-tpcc.md b/benchmark/v5.4-performance-benchmarking-with-tpcc.md index a9a02a3dfe053..05be43f4ca314 100644 --- a/benchmark/v5.4-performance-benchmarking-with-tpcc.md +++ b/benchmark/v5.4-performance-benchmarking-with-tpcc.md @@ -1,5 +1,6 @@ --- title: TiDB TPC-C Performance Test Report -- v5.4.0 vs. v5.3.0 +summary: TiDB v5.4.0 TPC-C performance is 3.16% better than v5.3.0. The improvement is consistent across different thread counts 2.80% (50 threads), 4.27% (100 threads), 3.45% (200 threads), and 2.11% (400 threads). --- # TiDB TPC-C Performance Test Report -- v5.4.0 vs. v5.3.0 diff --git a/benchmark/v5.4-performance-benchmarking-with-tpch.md b/benchmark/v5.4-performance-benchmarking-with-tpch.md index 87d7e82379a38..c459c56268f35 100644 --- a/benchmark/v5.4-performance-benchmarking-with-tpch.md +++ b/benchmark/v5.4-performance-benchmarking-with-tpch.md @@ -1,5 +1,6 @@ --- title: TiDB TPC-H Performance Test Report -- v5.4 MPP mode vs. Greenplum 6.15.0 and Apache Spark 3.1.1 +summary: TiDB v5.4 MPP mode outperforms Greenplum 6.15.0 and Apache Spark 3.1.1 in TPC-H 100 GB performance test. TiDB's MPP mode is 2-3 times faster. Test results show TiDB v5.4 has significantly lower query execution times compared to Greenplum and Apache Spark. --- # TiDB TPC-H Performance Test Report -- TiDB v5.4 MPP mode vs. Greenplum 6.15.0 and Apache Spark 3.1.1 diff --git a/benchmark/v6.0-performance-benchmarking-with-tpcc.md b/benchmark/v6.0-performance-benchmarking-with-tpcc.md index 818ac707dd01e..e8fa4c4a9e278 100644 --- a/benchmark/v6.0-performance-benchmarking-with-tpcc.md +++ b/benchmark/v6.0-performance-benchmarking-with-tpcc.md @@ -1,5 +1,6 @@ --- title: TiDB TPC-C Performance Test Report -- v6.0.0 vs. v5.4.0 +summary: TiDB v6.0.0 TPC-C performance is 24.20% better than v5.4.0. The improvement is consistent across different thread counts, with the highest improvement at 26.97% for 100 threads. --- # TiDB TPC-C Performance Test Report -- v6.0.0 vs. v5.4.0 diff --git a/benchmark/v6.0-performance-benchmarking-with-tpch.md b/benchmark/v6.0-performance-benchmarking-with-tpch.md index 1f59b4201348b..b22e4e3b0afab 100644 --- a/benchmark/v6.0-performance-benchmarking-with-tpch.md +++ b/benchmark/v6.0-performance-benchmarking-with-tpch.md @@ -1,5 +1,6 @@ --- title: Performance Comparison between TiFlash and Greenplum/Spark +summary: Performance Comparison between TiFlash and Greenplum/Spark. Refer to TiDB v5.4 TPC-H performance benchmarking report for details. --- # Performance Comparison between TiFlash and Greenplum/Spark diff --git a/benchmark/v6.1-performance-benchmarking-with-tpcc.md b/benchmark/v6.1-performance-benchmarking-with-tpcc.md index 1b51ad41e9113..17265b670510b 100644 --- a/benchmark/v6.1-performance-benchmarking-with-tpcc.md +++ b/benchmark/v6.1-performance-benchmarking-with-tpcc.md @@ -1,5 +1,6 @@ --- title: TiDB TPC-C Performance Test Report -- v6.1.0 vs. v6.0.0 +summary: TiDB v6.1.0 TPC-C performance is 2.85% better than v6.0.0. TiDB and TiKV parameter configurations are the same for both versions. HAProxy is used for load balancing. Results show performance improvement across different thread counts. --- # TiDB TPC-C Performance Test Report -- v6.1.0 vs. v6.0.0 diff --git a/benchmark/v6.1-performance-benchmarking-with-tpch.md b/benchmark/v6.1-performance-benchmarking-with-tpch.md index 1f59b4201348b..b22e4e3b0afab 100644 --- a/benchmark/v6.1-performance-benchmarking-with-tpch.md +++ b/benchmark/v6.1-performance-benchmarking-with-tpch.md @@ -1,5 +1,6 @@ --- title: Performance Comparison between TiFlash and Greenplum/Spark +summary: Performance Comparison between TiFlash and Greenplum/Spark. Refer to TiDB v5.4 TPC-H performance benchmarking report for details. --- # Performance Comparison between TiFlash and Greenplum/Spark diff --git a/benchmark/v6.2-performance-benchmarking-with-tpcc.md b/benchmark/v6.2-performance-benchmarking-with-tpcc.md index 81a90cd072e81..455f3ca53a90f 100644 --- a/benchmark/v6.2-performance-benchmarking-with-tpcc.md +++ b/benchmark/v6.2-performance-benchmarking-with-tpcc.md @@ -1,5 +1,6 @@ --- title: TiDB TPC-C Performance Test Report -- v6.2.0 vs. v6.1.0 +summary: TiDB v6.2.0 TPC-C performance declined by 2.00% compared to v6.1.0. The test used AWS EC2 with specific hardware and software configurations. Test data was prepared and stress tests were conducted via HAProxy. Results showed a decline in performance across different thread counts. --- # TiDB TPC-C Performance Test Report -- v6.2.0 vs. v6.1.0 diff --git a/benchmark/v6.2-performance-benchmarking-with-tpch.md b/benchmark/v6.2-performance-benchmarking-with-tpch.md index a3a0ad71b771b..834a7bb3276bf 100644 --- a/benchmark/v6.2-performance-benchmarking-with-tpch.md +++ b/benchmark/v6.2-performance-benchmarking-with-tpch.md @@ -1,5 +1,6 @@ --- title: Performance Comparison between TiFlash and Greenplum/Spark +summary: Performance Comparison between TiFlash and Greenplum/Spark. Refer to TiDB v5.4 TPC-H performance benchmarking report at the provided link. --- # Performance Comparison between TiFlash and Greenplum/Spark diff --git a/best-practices-for-security-configuration.md b/best-practices-for-security-configuration.md new file mode 100644 index 0000000000000..da461d8fea07f --- /dev/null +++ b/best-practices-for-security-configuration.md @@ -0,0 +1,124 @@ +--- +title: Best Practices for TiDB Security Configuration +summary: Learn the best practices for TiDB security configuration to help mitigate potential security risks. +--- + +# Best Practices for TiDB Security Configuration + +The security of TiDB is crucial for protecting data integrity and confidentiality. This document provides guidelines for configuring TiDB clusters securely during deployment. By following these best practices, you can effectively reduce potential security risks, prevent data breaches, and ensure the continuous, stable, and reliable operation of your TiDB database system. + +> **Note:** +> +> This document offers general recommendations on TiDB security configurations. PingCAP does not guarantee the completeness or accuracy of the information, and it assumes no responsibility for any issues arising from the use of this guide. Users should assess these recommendations based on their specific needs and consult professionals for tailored advice. + +## Set the initial password for the root user + +By default, the root user in a newly created TiDB cluster has no password, which poses a potential security risk. If a password is not set, anyone can attempt to log in to the TiDB database as the root user, potentially gaining access to and modifying data. + +To avoid this risk, it is recommended to set a root password during deployment: + +- For deployments using TiUP, refer to [Deploy TiDB Cluster Using TiUP](/production-deployment-using-tiup.md#step-7-start-a-tidb-cluster) to generate a random password for the root user. +- For deployments using TiDB Operator, refer to [Set initial account and password](https://docs.pingcap.com/tidb-in-kubernetes/stable/initialize-a-cluster#set-initial-account-and-password) to set the root password. + +## Enable password complexity checks + +By default, TiDB does not enforce password complexity policies, which might lead to the use of weak or empty passwords, increasing security risks. + +To ensure that database users create strong passwords, it is recommended to configure a reasonable [password complexity policy](/password-management.md#password-complexity-policy). For example, configure a policy that requires passwords to include a combination of uppercase letters, lowercase letters, numbers, and special characters. By enforcing password complexity checks, you can improve database security, prevent brute force attacks, reduce internal threats, ensure compliance with regulations, and lower the risk of data breaches, thus enhancing overall security. + +## Change the default Grafana password + +TiDB installation includes the Grafana component by default, and the default username and password are typically `admin`/`admin`. If the password is not changed promptly, attackers could exploit this to gain control of the system. + +It is recommended to immediately change the Grafana password to a strong one during the TiDB deployment, and regularly update the password to ensure system security. Here are the steps to change the Grafana password: + +- Upon first login to Grafana, follow the prompts to change the password. + + ![Grafana Password Reset Guide](/media/grafana-password-reset1.png) + +- Access the Grafana personal configuration center to change the password. + + ![Grafana Password Reset Guide](/media/grafana-password-reset2.png) + +## Enhance TiDB Dashboard security + +### Use a least privilege user + +TiDB Dashboard shares the account system with TiDB SQL users, and TiDB Dashboard authorization is based on TiDB SQL user permissions. TiDB Dashboard requires minimal permissions and can even operate with read-only access. + +To enhance security, it is recommended to create a [least-privilege SQL user](/dashboard/dashboard-user.md) for accessing the TiDB Dashboard and to avoid using high-privilege users. + +### Restrict access control + +By default, TiDB Dashboard is designed for trusted users. The default port includes additional API interfaces besides TiDB Dashboard. If you want to allow access to TiDB Dashboard from external networks or untrusted users, take the following measures to avoid security vulnerabilities: + +- Use a firewall or other mechanisms to restrict the default `2379` port to trusted domains, preventing access by external users. + + > **Note:** + > + > TiDB, TiKV, and other components need to communicate with the PD component via the PD client port. Do not block internal network access between components, which will make the cluster unavailable. + +- [Configure a reverse proxy](/dashboard/dashboard-ops-reverse-proxy.md#use-tidb-dashboard-behind-a-reverse-proxy) to securely provide TiDB Dashboard services to external users on a different port. + +## Protect internal ports + +By default, TiDB installation includes several privileged interfaces for inter-component communication. These ports typically do not need to be accessible to users, because they are primarily for internal communication. Exposing these ports on public networks increases the attack surface, violates the principle of least privilege, and raises the risk of security vulnerabilities. The following table lists the default listening ports in a TiDB cluster: + +| Component | Default port| Protocol | +|-------------------|-------------|------------| +| TiDB | 4000 | MySQL | +| TiDB | 10080 | HTTP | +| TiKV | 20160 | Protocol | +| TiKV | 20180 | HTTP | +| PD | 2379 | HTTP/Protocol| +| PD | 2380 | Protocol | +| TiFlash | 3930 | Protocol | +| TiFlash | 20170 | Protocol | +| TiFlash | 20292 | HTTP | +| TiFlash | 8234 | HTTP | +| DM master | 8261 | HTTP | +| DM master | 8291 | HTTP | +| DM worker | 8262 | HTTP | +| TiCDC | 8300 | HTTP | +| TiDB Lightning | 8289 | HTTP | +| TiDB Operator | 6060 | HTTP | +| TiDB Dashboard | 2379 | HTTP | +| TiDB Binlog | 8250 | HTTP | +| TiDB Binlog | 8249 | HTTP | +| TMS | 8082 | HTTP | +| TEM | 8080 | HTTP | +| TEM | 8000 | HTTP | +| TEM | 4110 | HTTP | +| TEM | 4111 | HTTP | +| TEM | 4112 | HTTP | +| TEM | 4113 | HTTP | +| TEM | 4124 | HTTP | +| Prometheus | 9090 | HTTP | +| Grafana | 3000 | HTTP | +| AlertManager | 9093 | HTTP | +| AlertManager | 9094 | Protocol | +| Node Exporter | 9100 | HTTP | +| Blackbox Exporter | 9115 | HTTP | +| NG Monitoring | 12020 | HTTP | + +It is recommended to only expose the `4000` port for the database and the `9000` port for the Grafana dashboard to ordinary users, while restricting access to other ports using network security policies or firewalls. The following is an example of using `iptables` to restrict port access: + +```shell +# Allow internal port communication from the whitelist of component IP addresses +sudo iptables -A INPUT -s internal IP address range -j ACCEPT + +# Only open ports 4000 and 9000 to external users +sudo iptables -A INPUT -p tcp --dport 4000 -j ACCEPT +sudo iptables -A INPUT -p tcp --dport 9000 -j ACCEPT + +# Deny all other traffic by default +sudo iptables -P INPUT DROP +``` + +If you need to access TiDB Dashboard, it is recommended to [configure a reverse proxy](/dashboard/dashboard-ops-reverse-proxy.md#use-tidb-dashboard-behind-a-reverse-proxy) to securely provide services to external networks on a separate port. + +## Resolving false positives from third-party MySQL vulnerability scanners + +Most vulnerability scanners detect MySQL vulnerabilities based on version information. Because TiDB is MySQL protocol-compatible but not MySQL itself, version-based vulnerability scans might lead to false positives. It is recommended to focus vulnerability scans on principle-based assessments. If compliance scanning tools require a specific MySQL version, you can [modify the server version number](/faq/high-reliability-faq.md#does-tidb-support-modifying-the-mysql-version-string-of-the-server-to-a-specific-one-that-is-required-by-the-security-vulnerability-scanning-tool) to meet the requirement. + +By changing the server version number, you can avoid false positives from vulnerability scanners. The [`server-version`](/tidb-configuration-file.md#server-version) value is used by TiDB nodes to verify the current TiDB version. Before upgrading the TiDB cluster, ensure that the `server-version` value is either empty or the actual version of TiDB to avoid unexpected behavior. \ No newline at end of file diff --git a/best-practices-on-public-cloud.md b/best-practices/best-practices-on-public-cloud.md similarity index 88% rename from best-practices-on-public-cloud.md rename to best-practices/best-practices-on-public-cloud.md index d4677de975273..764ba1ccc462c 100644 --- a/best-practices-on-public-cloud.md +++ b/best-practices/best-practices-on-public-cloud.md @@ -7,7 +7,40 @@ summary: Learn about the best practices for deploying TiDB on public cloud. Public cloud infrastructure has become an increasingly popular choice for deploying and managing TiDB. However, deploying TiDB on public cloud requires careful consideration of several critical factors, including performance tuning, cost optimization, reliability, and scalability. -This document covers various essential best practices for deploying TiDB on public cloud, such as using a dedicated disk for Raft Engine, reducing compaction I/O flow in KV RocksDB, optimizing costs for cross-AZ traffic, mitigating Google Cloud live migration events, and fine-tuning the PD server in large clusters. By following these best practices, you can maximize the performance, cost efficiency, reliability, and scalability of your TiDB deployment on public cloud. +This document covers various essential best practices for deploying TiDB on public cloud, such as reducing compaction I/O flow in KV RocksDB, using a dedicated disk for Raft Engine, optimizing costs for cross-AZ traffic, mitigating Google Cloud live migration events, and fine-tuning the PD server in large clusters. By following these best practices, you can maximize the performance, cost efficiency, reliability, and scalability of your TiDB deployment on public cloud. + +## Reduce compaction I/O flow in KV RocksDB + +As the storage engine of TiKV, [RocksDB](https://rocksdb.org/) is used to store user data. Because the provisioned IO throughput on cloud EBS is usually limited due to cost considerations, RocksDB might exhibit high write amplification, and the disk throughput might become the bottleneck for the workload. As a result, the total number of pending compaction bytes grows over time and triggers flow control, which indicates that TiKV lacks sufficient disk bandwidth to keep up with the foreground write flow. + +To alleviate the bottleneck caused by limited disk throughput, you can improve performance by [enabling Titan](#enable-titan). If your average row size is smaller than 512 bytes, Titan is not applicable. In this case, you can improve performance by [increasing all the compression levels](#increase-all-the-compression-levels). + +### Enable Titan + +[Titan](/storage-engine/titan-overview.md) is a high-performance [RocksDB](https://github.com/facebook/rocksdb) plugin for key-value separation, which can reduce write amplification in RocksDB when large values are used. + +If your average row size is larger than 512 bytes, you can enable Titan to reduce the compaction I/O flow as follows, with `min-blob-size` set to `"512B"` or `"1KB"` and `blob-file-compression` set to `"zstd"`: + +```toml +[rocksdb.titan] +enabled = true +[rocksdb.defaultcf.titan] +min-blob-size = "1KB" +blob-file-compression = "zstd" +``` + +> **Note:** +> +> When Titan is enabled, there might be a slight performance degradation for range scans on the primary key. For more information, see [Impact of `min-blob-size` on performance](https://docs.pingcap.com/tidb/stable/titan-overview#impact-of-min-blob-size-on-performance). + +### Increase all the compression levels + +If your average row size is smaller than 512 bytes, you can increase all the compression levels of the default column family to `"zstd"` as follows: + +```toml +[rocksdb.defaultcf] +compression-per-level = ["zstd", "zstd", "zstd", "zstd", "zstd", "zstd", "zstd"] +``` ## Use a dedicated disk for Raft Engine @@ -97,31 +130,12 @@ tikv: storageSize: 512Gi ``` -## Reduce compaction I/O flow in KV RocksDB - -As the storage engine of TiKV, [RocksDB](https://rocksdb.org/) is used to store user data. Because the provisioned IO throughput on cloud EBS is usually limited due to cost considerations, RocksDB might exhibit high write amplification, and the disk throughput might become the bottleneck for the workload. As a result, the total number of pending compaction bytes grows over time and triggers flow control, which indicates that TiKV lacks sufficient disk bandwidth to keep up with the foreground write flow. - -To alleviate the bottleneck caused by limited disk throughput, you can improve performance by increasing the compression level for RocksDB and reducing the disk throughput. For example, you can refer to the following example to increase all the compression levels of the default column family to `zstd`. - -``` -[rocksdb.defaultcf] -compression-per-level = ["zstd", "zstd", "zstd", "zstd", "zstd", "zstd", "zstd"] -``` - ## Optimize cost for cross-AZ network traffic Deploying TiDB across multiple availability zones (AZs) can lead to increased costs due to cross-AZ data transfer fees. To optimize costs, it is important to reduce cross-AZ network traffic. To reduce cross-AZ read traffic, you can enable the [Follower Read feature](/follower-read.md), which allows TiDB to prioritize selecting replicas in the same availability zone. To enable this feature, set the [`tidb_replica_read`](/system-variables.md#tidb_replica_read-new-in-v40) variable to `closest-replicas` or `closest-adaptive`. -To reduce cross-AZ write traffic in TiKV instances, you can enable the gRPC compression feature, which compresses data before transmitting it over the network. The following configuration example shows how to enable gzip gRPC compression for TiKV. - -``` -server_configs: - tikv: - server.grpc-compression-type: gzip -``` - To reduce network traffic caused by the data shuffle of TiFlash MPP tasks, it is recommended to deploy multiple TiFlash instances in the same availability zones (AZs). Starting from v6.6.0, [compression exchange](/explain-mpp.md#mpp-version-and-exchange-data-compression) is enabled by default, which reduces the network traffic caused by MPP data shuffle. ## Mitigate live migration maintenance events on Google Cloud @@ -134,7 +148,7 @@ To detect live migration events initiated by Google Cloud and mitigate the perfo - TiKV: Evicts leaders on the affected TiKV store during maintenance. - PD: Resigns a leader if the current PD instance is the PD leader. -It is important to note that this watching script is specifically designed for TiDB clusters deployed using [TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/dev/tidb-operator-overview), which offers enhanced management functionalities for TiDB in Kubernetes environments. +It is important to note that this watching script is specifically designed for TiDB clusters deployed using [TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/v1.5/tidb-operator-overview), which offers enhanced management functionalities for TiDB in Kubernetes environments. By utilizing the watching script and taking necessary actions during maintenance events, TiDB clusters can better handle live migration events on Google Cloud and ensure smoother operations with minimal impact on query processing and response times. @@ -180,7 +194,7 @@ To reduce the number of Regions and alleviate the heartbeat overhead on the syst ## After tuning -After the tunning, the following effects can be observed: +After the tuning, the following effects can be observed: - The TSO requests per second are decreased to 64,800. - The CPU utilization is significantly reduced from approximately 4,600% to 1,400%. diff --git a/ddl-introduction.md b/best-practices/ddl-introduction.md similarity index 97% rename from ddl-introduction.md rename to best-practices/ddl-introduction.md index 079089e448dd3..7658ccfae57e7 100644 --- a/ddl-introduction.md +++ b/best-practices/ddl-introduction.md @@ -29,7 +29,7 @@ Based on whether to operate the data included in the target DDL object, DDL stat In TiDB, physical DDL statements are also referred to as "reorg DDL", which stands for reorganization. Currently, physical DDL statements only include `ADD INDEX` and lossy column type changes (such as changing from an `INT` type to a `CHAR` type). These statements take a long time to execute, and the execution time is affected by the amount of data in the table, the machine configuration, and the application workload. - Executing physical DDL statements can have an impact on the workload of the application for two reasons. On the one hand, it consumes CPU and I/O resources from TiKV to read data and write new data. On the other hand, the TiDB node where the DDL Owner is located needs to perform the corresponding computations, which consumes more CPU resources. Because TiDB does not support distributed execution of DDL statements, other TiDB nodes do not consume additional system resources during this process. + Executing physical DDL statements can have an impact on the workload of the application for two reasons. On the one hand, it consumes CPU and I/O resources from TiKV to read data and write new data. On the other hand, **the TiDB node serving as DDL Owners** or **those TiDB nodes scheduled by the TiDB Distributed eXecution Framework (DXF) to execute `ADD INDEX` tasks** consume CPU resources from TiDB to perform the corresponding computations. > **Note:** > @@ -77,7 +77,7 @@ absent -> delete only -> write only -> write reorg -> public For users, the newly created index is unavailable before the `public` state. -
+
Before v6.2.0, the process of handling asynchronous schema changes in the TiDB SQL layer is as follows: diff --git a/best-practices/grafana-monitor-best-practices.md b/best-practices/grafana-monitor-best-practices.md index 5fb569a29d538..206b1f5334b60 100644 --- a/best-practices/grafana-monitor-best-practices.md +++ b/best-practices/grafana-monitor-best-practices.md @@ -1,7 +1,6 @@ --- title: Best Practices for Monitoring TiDB Using Grafana -summary: Learn seven tips for efficiently using Grafana to monitor TiDB. -aliases: ['/docs/dev/best-practices/grafana-monitor-best-practices/','/docs/dev/reference/best-practices/grafana-monitor/'] +summary: Best Practices for Monitoring TiDB Using Grafana. Deploy a TiDB cluster using TiUP and add Grafana and Prometheus for monitoring. Use metrics to analyze cluster status and diagnose problems. Prometheus collects metrics from TiDB components, and Grafana displays them. Tips for efficient Grafana use include modifying query expressions, switching Y-axis scale, and using API for query results. The platform is powerful for analyzing and diagnosing TiDB cluster status. --- # Best Practices for Monitoring TiDB Using Grafana @@ -22,12 +21,12 @@ For TiDB 2.1.3 or later versions, TiDB monitoring supports the pull method. It i ## Source and display of monitoring data -The three core components of TiDB (TiDB server, TiKV server and PD server) obtain metrics through the HTTP interface. These metrics are collected from the program code, and the ports are as follows: +The three core components of TiDB (TiDB server, TiKV server and PD server) obtain metrics through the HTTP interface. These metrics are collected from the program code, and the default ports are as follows: | Component | Port | | :---------- |:----- | | TiDB server | 10080 | -| TiKV server | 20181 | +| TiKV server | 20180 | | PD server | 2379 | Execute the following command to check the QPS of a SQL statement through the HTTP interface. Take the TiDB server as an example: @@ -160,7 +159,7 @@ The API of Prometheus is shown as follows: {{< copyable "shell-regular" >}} ```bash -curl -u user:pass 'http://__grafana_ip__:3000/api/datasources/proxy/1/api/v1/query_range?query=sum(tikv_engine_size_bytes%7Binstancexxxxxxxxx20181%22%7D)%20by%20(instance)&start=1565879269&end=1565882869&step=30' |python -m json.tool +curl -u user:pass 'http://__grafana_ip__:3000/api/datasources/proxy/1/api/v1/query_range?query=sum(tikv_engine_size_bytes%7Binstancexxxxxxxxx20180%22%7D)%20by%20(instance)&start=1565879269&end=1565882869&step=30' |python -m json.tool ``` ``` @@ -169,7 +168,7 @@ curl -u user:pass 'http://__grafana_ip__:3000/api/datasources/proxy/1/api/v1/que "result": [ { "metric": { - "instance": "xxxxxxxxxx:20181" + "instance": "xxxxxxxxxx:20180" }, "values": [ [ diff --git a/best-practices/haproxy-best-practices.md b/best-practices/haproxy-best-practices.md index cc01bbb7ccf48..22957bf5077a3 100644 --- a/best-practices/haproxy-best-practices.md +++ b/best-practices/haproxy-best-practices.md @@ -1,7 +1,6 @@ --- title: Best Practices for Using HAProxy in TiDB -summary: This document describes best practices for configuration and usage of HAProxy in TiDB. -aliases: ['/docs/dev/best-practices/haproxy-best-practices/','/docs/dev/reference/best-practices/haproxy/'] +summary: HAProxy is a free, open-source load balancer and proxy server for TCP and HTTP-based applications. It provides high availability, load balancing, health checks, sticky sessions, SSL support, and monitoring. To deploy HAProxy, ensure hardware and software requirements are met, then install and configure it. Use the latest stable version for best results. --- # Best Practices for Using HAProxy in TiDB @@ -35,12 +34,12 @@ Before you deploy HAProxy, make sure that you meet the hardware and software req ### Hardware requirements -For your server, it is recommended to meet the following hardware requirements. You can also improve server specifications according to the load balancing environment. +According to the [HAProxy documentation](https://www.haproxy.com/documentation/haproxy-enterprise/getting-started/installation/linux/), the minimum hardware configuration for HAProxy is shown in the following table. Under the Sysbench `oltp_read_write` workload, the maximum QPS for this configuration is about 50K. You can increase the server configuration according to your load balancing environment. | Hardware resource | Minimum specification | | :--------------------- | :-------------------- | | CPU | 2 cores, 3.5 GHz | -| Memory | 16 GB | +| Memory | 4 GB | | Storage | 50 GB (SATA) | | Network Interface Card | 10G Network Card | @@ -213,7 +212,7 @@ listen tidb-cluster # Database load balancing. To check the source IP address using `SHOW PROCESSLIST`, you need to configure the [PROXY protocol](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt) to connect to TiDB. ```yaml - server tidb-1 10.9.18.229:4000 send-proxy check inter 2000 rise 2 fall 3 + server tidb-1 10.9.18.229:4000 send-proxy check inter 2000 rise 2 fall 3 server tidb-2 10.9.39.208:4000 send-proxy check inter 2000 rise 2 fall 3 server tidb-3 10.9.64.166:4000 send-proxy check inter 2000 rise 2 fall 3 ``` diff --git a/best-practices/high-concurrency-best-practices.md b/best-practices/high-concurrency-best-practices.md index 16e6df2ce2543..b6894632819b1 100644 --- a/best-practices/high-concurrency-best-practices.md +++ b/best-practices/high-concurrency-best-practices.md @@ -1,7 +1,6 @@ --- title: Highly Concurrent Write Best Practices -summary: Learn best practices for highly-concurrent write-intensive workloads in TiDB. -aliases: ['/docs/dev/best-practices/high-concurrency-best-practices/','/docs/dev/reference/best-practices/high-concurrency/'] +summary: This document provides best practices for handling highly-concurrent write-heavy workloads in TiDB. It addresses challenges and solutions for data distribution, hotspot cases, and complex hotspot problems. The article also discusses parameter configuration for optimizing performance. --- # Highly Concurrent Write Best Practices @@ -10,11 +9,11 @@ This document describes best practices for handling highly-concurrent write-heav ## Target audience -This document assumes that you have a basic understanding of TiDB. It is recommended that you first read the following three blog articles that explain TiDB fundamentals, and [TiDB Best Practices](https://en.pingcap.com/blog/tidb-best-practice/): +This document assumes that you have a basic understanding of TiDB. It is recommended that you first read the following three blog articles that explain TiDB fundamentals, and [TiDB Best Practices](https://www.pingcap.com/blog/tidb-best-practice/): -+ [Data Storage](https://en.pingcap.com/blog/tidb-internal-data-storage/) -+ [Computing](https://en.pingcap.com/blog/tidb-internal-computing/) -+ [Scheduling](https://en.pingcap.com/blog/tidb-internal-scheduling/) ++ [Data Storage](https://www.pingcap.com/blog/tidb-internal-data-storage/) ++ [Computing](https://www.pingcap.com/blog/tidb-internal-computing/) ++ [Scheduling](https://www.pingcap.com/blog/tidb-internal-scheduling/) ## Highly-concurrent write-intensive scenario @@ -33,7 +32,7 @@ For a distributed database, it is important to make full use of the capacity of ## Data distribution principles in TiDB -To address the above challenges, it is necessary to start with the data segmentation and scheduling principle of TiDB. Refer to [Scheduling](https://en.pingcap.com/blog/tidb-internal-scheduling/) for more details. +To address the above challenges, it is necessary to start with the data segmentation and scheduling principle of TiDB. Refer to [Scheduling](https://www.pingcap.com/blog/tidb-internal-scheduling/) for more details. TiDB splits data into Regions, each representing a range of data with a size limit of 96M by default. Each Region has multiple replicas, and each group of replicas is called a Raft Group. In a Raft Group, the Region Leader executes the read and write tasks (TiDB supports [Follower-Read](/follower-read.md)) within the data range. The Region Leader is automatically scheduled by the Placement Driver (PD) component to different physical nodes evenly to distribute the read and write pressure. @@ -80,10 +79,10 @@ SELECT '@example.com' ) FROM - (WITH RECURSIVE nr(n) AS + (WITH RECURSIVE nr(n) AS (SELECT 1 -- Start CTE at 1 UNION ALL SELECT n + 1 -- increase n with 1 every loop - FROM nr WHERE n < 1000000 -- stop loop at 1_000_000 + FROM nr WHERE n < 1000000 -- stop loop at 1_000_000 ) SELECT n FROM nr ) a; ``` diff --git a/best-practices/massive-regions-best-practices.md b/best-practices/massive-regions-best-practices.md index 5f73abdbacefd..8b1effde4b944 100644 --- a/best-practices/massive-regions-best-practices.md +++ b/best-practices/massive-regions-best-practices.md @@ -1,7 +1,6 @@ --- title: Best Practices for TiKV Performance Tuning with Massive Regions -summary: Learn how to tune the performance of TiKV with a massive amount of Regions. -aliases: ['/docs/dev/best-practices/massive-regions-best-practices/','/docs/dev/reference/best-practices/massive-regions/'] +summary: TiKV performance tuning involves reducing the number of Regions and messages, increasing Raftstore concurrency, enabling Hibernate Region and Region Merge, adjusting Raft base tick interval, increasing TiKV instances, and adjusting Region size. Other issues include slow PD leader switching and outdated PD routing information. --- # Best Practices for TiKV Performance Tuning with Massive Regions @@ -51,6 +50,18 @@ You can check the following monitoring metrics in Grafana's **TiKV Dashboard**: ![Check Propose wait duration](/media/best-practices/propose-wait-duration.png) ++ `Commit log duration` in the **Raft IO** panel + + `Commit log duration` is the time Raftstore takes to commit Raft logs to the majority of members in the respective Region. The possible reasons for a high value of this metric with significant fluctuations include the following: + + - The workload on Raftstore is heavy. + - The append log operation is slow. + - Raft logs cannot be committed timely due to network congestion. + + Reference value: lower than 200-500 ms. + + ![Check Commit log duration](/media/best-practices/commit-log-duration.png) + ## Performance tuning methods After finding out the cause of a performance problem, try to solve it from the following two aspects: @@ -66,7 +77,7 @@ By default, `raftstore.store-pool-size` is configured to `2` in TiKV. If a bottl ### Method 2: Enable Hibernate Region -In the actual situation, read and write requests are not evenly distributed on every Region. Instead, they are concentrated on a few Regions. Then you can minimize the number of messages between the Raft leader and the followers for the temporarily idle Regions, which is the feature of Hibernate Region. In this feature, Raftstore does sent tick messages to the Raft state machines of idle Regions if not necessary. Then these Raft state machines will not be triggered to generate heartbeat messages, which can greatly reduce the workload of Raftstore. +In the actual situation, read and write requests are not evenly distributed on every Region. Instead, they are concentrated on a few Regions. Then you can minimize the number of messages between the Raft leader and the followers for the temporarily idle Regions, which is the feature of Hibernate Region. In this feature, Raftstore doesn't send tick messages to the Raft state machines of idle Regions if not necessary. Then these Raft state machines will not be triggered to generate heartbeat messages, which can greatly reduce the workload of Raftstore. Hibernate Region is enabled by default in [TiKV master](https://github.com/tikv/tikv/tree/master). You can configure this feature according to your needs. For details, refer to [Configure Hibernate Region](/tikv-configuration-file.md). @@ -128,13 +139,17 @@ If Region followers have not received the heartbeat from the leader within the ` The default size of a Region is 96 MiB, and you can reduce the number of Regions by setting Regions to a larger size. For more information, see [Tune Region Performance](/tune-region-performance.md). -> **Warning:** +> **Note:** > -> Currently, customized Region size is an experimental feature introduced in TiDB v6.1.0. It is not recommended that you use it in production environments. The risks are as follows: +> Customized Region size is an experimental feature before TiDB v6.5.0. If you need to resize the Region size, it is recommended that you upgrade to v6.5.0 or a later version. + +### Method 7: Increase the maximum number of connections for Raft communication + +By default, the maximum number of connections used for Raft communication between TiKV nodes is 1. Increasing this number can help alleviate blockage issues caused by heavy communication workloads of a large number of Regions. For detailed instructions, see [`grpc-raft-conn-num`](/tikv-configuration-file.md#grpc-raft-conn-num). + +> **Note:** > -> + Performance jitter might be caused. -> + The query performance, especially for queries that deal with a large range of data, might decrease. -> + The Region scheduling slows down. +> To reduce unnecessary thread switching overhead and mitigate potential negative impacts from batch processing, it is recommended to set the number within the range of `[1, 4]`. ## Other problems and solutions diff --git a/best-practices/pd-scheduling-best-practices.md b/best-practices/pd-scheduling-best-practices.md index d8add90c1688d..dd7783dcb490e 100644 --- a/best-practices/pd-scheduling-best-practices.md +++ b/best-practices/pd-scheduling-best-practices.md @@ -1,7 +1,6 @@ --- title: PD Scheduling Best Practices -summary: Learn best practice and strategy for PD scheduling. -aliases: ['/docs/dev/best-practices/pd-scheduling-best-practices/','/docs/dev/reference/best-practices/pd-scheduling/'] +summary: This document summarizes PD scheduling best practices, including scheduling process, load balancing, hot regions scheduling, cluster topology awareness, scale-down and failure recovery, region merge, query scheduling status, and control scheduling strategy. It also covers common scenarios such as uneven distribution of leaders/regions, slow node recovery, and troubleshooting TiKV nodes. --- # PD Scheduling Best Practices @@ -254,7 +253,7 @@ Hot regions scheduling issues generally fall into the following categories: - The load of some nodes is significantly higher than that of other nodes from TiKV-related metrics, which becomes the bottleneck of the whole system. Currently, PD counts hotspots through traffic analysis only, so it is possible that PD fails to identify hotspots in certain scenarios. For example, when there are intensive point lookup requests for some regions, it might not be obvious to detect in traffic, but still the high QPS might lead to bottlenecks in key modules. - **Solutions**: Firstly, locate the table where hot regions are formed based on the specific business. Then add a `scatter-range-scheduler` scheduler to make all regions of this table evenly distributed. TiDB also provides an interface in its HTTP API to simplify this operation. Refer to [TiDB HTTP API](https://github.com/pingcap/tidb/blob/master/docs/tidb_http_api.md) for more details. + **Solutions**: Firstly, locate the table where hot regions are formed based on the specific business. Then add a `scatter-range-scheduler` scheduler to make all regions of this table evenly distributed. TiDB also provides an interface in its HTTP API to simplify this operation. Refer to [TiDB HTTP API](https://github.com/pingcap/tidb/blob/release-7.5/docs/tidb_http_api.md) for more details. ### Region merge is slow @@ -297,4 +296,8 @@ If a TiKV node fails, PD defaults to setting the corresponding node to the **dow Practically, if a node failure is considered unrecoverable, you can immediately take it offline. This makes PD replenish replicas soon in another node and reduces the risk of data loss. In contrast, if a node is considered recoverable, but the recovery cannot be done in 30 minutes, you can temporarily adjust `max-store-down-time` to a larger value to avoid unnecessary replenishment of the replicas and resources waste after the timeout. -In TiDB v5.2.0, TiKV introduces the mechanism of slow TiKV node detection. By sampling the requests in TiKV, this mechanism works out a score ranging from 1 to 100. A TiKV node with a score higher than or equal to 80 is marked as slow. You can add [`evict-slow-store-scheduler`](/pd-control.md#scheduler-show--add--remove--pause--resume--config--describe) to detect and schedule slow nodes. If only one TiKV is detected as slow, and the slow score reaches the limit (80 by default), the leader in this node will be evicted (similar to the effect of `evict-leader-scheduler`). +In TiDB v5.2.0, TiKV introduces the mechanism of slow TiKV node detection. By sampling the requests in TiKV, this mechanism works out a score ranging from 1 to 100. A TiKV node with a score higher than or equal to 80 is marked as slow. You can add [`evict-slow-store-scheduler`](/pd-control.md#scheduler-show--add--remove--pause--resume--config--describe) to detect and schedule slow nodes. If only one TiKV is detected as slow, and the slow score reaches the limit (80 by default), the Leader in this node will be evicted (similar to the effect of `evict-leader-scheduler`). + +> **Note:** +> +> **Leader eviction** is accomplished by PD sending scheduling requests to TiKV slow nodes and then TiKV executing the received scheduling requests sequentially. Due to factors such as **slow I/O**, slow nodes might experience request accumulation, causing some Leaders to wait until the delayed requests are processed before handling **Leader eviction** requests. This results in an overall extended time for **Leader eviction**. Therefore, when you enable `evict-slow-store-scheduler`, it is recommended to enable [`store-io-pool-size`](/tikv-configuration-file.md#store-io-pool-size-new-in-v530) as well to mitigate this situation. diff --git a/best-practices/readonly-nodes.md b/best-practices/readonly-nodes.md index 0239f94c50aa2..9597e8ddbf6cf 100644 --- a/best-practices/readonly-nodes.md +++ b/best-practices/readonly-nodes.md @@ -1,6 +1,6 @@ --- title: Best Practices for Read-Only Storage Nodes -summary: Learn how to configure read-only storage nodes to physically isolate important online services. +summary: This document introduces configuring read-only storage nodes for isolating high-tolerance delay loads from online services. Steps include marking TiKV nodes as read-only, using Placement Rules to store data on read-only nodes as learners, and using Follower Read to read data from read-only nodes. --- # Best Practices for Read-Only Storage Nodes diff --git a/best-practices/three-dc-local-read.md b/best-practices/three-dc-local-read.md index 2b776172ac321..24c9001302036 100644 --- a/best-practices/three-dc-local-read.md +++ b/best-practices/three-dc-local-read.md @@ -1,6 +1,6 @@ --- title: Local Read under Three Data Centers Deployment -summary: Learn how to use the Stale Read feature to read local data under three DCs deployment and thus reduce cross-center requests. +summary: TiDB's three data center deployment model can cause increased access latency due to cross-center data reads. To mitigate this, the Stale Read feature allows for local historical data access, reducing latency at the expense of real-time data availability. When using Stale Read in geo-distributed scenarios, TiDB accesses local replicas to avoid cross-center network latency. This is achieved by configuring the `zone` label and setting `tidb_replica_read` to `closest-replicas`. For more information on performing Stale Read, refer to the documentation. --- # Local Read under Three Data Centers Deployment diff --git a/best-practices/three-nodes-hybrid-deployment.md b/best-practices/three-nodes-hybrid-deployment.md index 5c38598c0c02c..9c6cb82d7bcb8 100644 --- a/best-practices/three-nodes-hybrid-deployment.md +++ b/best-practices/three-nodes-hybrid-deployment.md @@ -1,6 +1,6 @@ --- title: Best Practices for Three-Node Hybrid Deployment -summary: Learn the best practices for three-node hybrid deployment. +summary: TiDB cluster can be deployed in a cost-effective way on three machines. Best practices for this hybrid deployment include adjusting parameters for stability and performance. Limiting resource consumption and adjusting thread pool sizes are key to optimizing the cluster. Adjusting parameters for TiKV background tasks and TiDB execution operators is also important. --- # Best Practices for Three-Node Hybrid Deployment diff --git a/best-practices/tidb-best-practices.md b/best-practices/tidb-best-practices.md index fe21b758386d1..a941c4e56d88b 100644 --- a/best-practices/tidb-best-practices.md +++ b/best-practices/tidb-best-practices.md @@ -1,7 +1,6 @@ --- title: TiDB Best Practices -summary: Learn the best practices of using TiDB. -aliases: ['/docs/dev/tidb-best-practices/'] +summary: This document summarizes best practices for using TiDB, covering SQL use and optimization tips for OLAP and OLTP scenarios, with a focus on TiDB-specific optimization options. It also recommends reading three blog posts introducing TiDB's technical principles before diving into the best practices. --- # TiDB Best Practices @@ -10,9 +9,9 @@ This document summarizes the best practices of using TiDB, including the use of Before you read this document, it is recommended that you read three blog posts that introduce the technical principles of TiDB: -* [TiDB Internal (I) - Data Storage](https://en.pingcap.com/blog/tidb-internal-data-storage/) -* [TiDB Internal (II) - Computing](https://en.pingcap.com/blog/tidb-internal-computing/) -* [TiDB Internal (III) - Scheduling](https://en.pingcap.com/blog/tidb-internal-scheduling/) +* [TiDB Internal (I) - Data Storage](https://www.pingcap.com/blog/tidb-internal-data-storage/) +* [TiDB Internal (II) - Computing](https://www.pingcap.com/blog/tidb-internal-computing/) +* [TiDB Internal (III) - Scheduling](https://www.pingcap.com/blog/tidb-internal-scheduling/) ## Preface @@ -34,7 +33,7 @@ To store three replicas, compared with the replication of Source-Replica, Raft i ### Distributed transactions -TiDB provides complete distributed transactions and the model has some optimizations on the basis of [Google Percolator](https://research.google.com/pubs/pub36726.html). This document introduces the following features: +TiDB provides complete distributed transactions and the model has some optimizations on the basis of [Google Percolator](https://research.google/pubs/large-scale-incremental-processing-using-distributed-transactions-and-notifications/). This document introduces the following features: * Optimistic transaction model @@ -68,7 +67,7 @@ Placement Driver (PD) balances the load of the cluster according to the status o ### SQL on KV -TiDB automatically maps the SQL structure into Key-Value structure. For details, see [TiDB Internal (II) - Computing](https://en.pingcap.com/blog/tidb-internal-computing/). +TiDB automatically maps the SQL structure into Key-Value structure. For details, see [TiDB Internal (II) - Computing](https://www.pingcap.com/blog/tidb-internal-computing/). Simply put, TiDB performs the following operations: diff --git a/best-practices/uuid.md b/best-practices/uuid.md index 34c6308db8222..406831b742ad3 100644 --- a/best-practices/uuid.md +++ b/best-practices/uuid.md @@ -1,6 +1,6 @@ --- title: UUID Best Practices -summary: Learn best practice and strategy for using UUIDs with TiDB. +summary: UUIDs, when used as primary keys, offer benefits such as reduced network trips, support in most programming languages and databases, and protection against enumeration attacks. Storing UUIDs as binary in a `BINARY(16)` column is recommended. It's also advised to avoid setting the `swap_flag` with TiDB to prevent hotspots. MySQL compatibility is available for UUIDs. --- # UUID Best Practices diff --git a/binary-package.md b/binary-package.md index 9bada3c1e8e48..31c5f33410d5a 100644 --- a/binary-package.md +++ b/binary-package.md @@ -5,7 +5,7 @@ summary: Learn about TiDB installation packages and the specific components incl # TiDB Installation Packages -Before [deploying TiUP offline](/production-deployment-using-tiup.md#deploy-tiup-offline), you need to download the binary packages of TiDB at the [official download page](https://en.pingcap.com/download/). +Before [deploying TiUP offline](/production-deployment-using-tiup.md#deploy-tiup-offline), you need to download the binary packages of TiDB as described in [Prepare the TiUP offline component package](/production-deployment-using-tiup.md#prepare-the-tiup-offline-component-package). TiDB binary packages are available in amd64 and arm64 architectures. In either architecture, TiDB provides two binary packages: `TiDB-community-server` and `TiDB-community-toolkit`. diff --git a/br/backup-and-restore-design.md b/br/backup-and-restore-design.md index 0e75cd4598532..7919417dc17ce 100644 --- a/br/backup-and-restore-design.md +++ b/br/backup-and-restore-design.md @@ -1,6 +1,6 @@ --- title: Overview of TiDB Backup & Restore Architecture -summary: Learn about the architecture design of TiDB backup and restore features. +summary: TiDB supports backup and restore for cluster data using Backup & Restore (BR) and TiDB Operator. Tasks can be created to back up data from TiKV nodes and restore data to TiKV nodes. The architecture includes full data backup and restore, data change log backup, and point-in-time recovery (PITR). For details, refer to specific documents for each feature. --- # Overview of TiDB Backup & Restore Architecture diff --git a/br/backup-and-restore-overview.md b/br/backup-and-restore-overview.md index 5c0ad2eed447c..b1bd3edac3430 100644 --- a/br/backup-and-restore-overview.md +++ b/br/backup-and-restore-overview.md @@ -1,7 +1,6 @@ --- title: TiDB Backup & Restore Overview -summary: Learn about the definition and features of TiDB Backup & Restore. -aliases: ['/docs/dev/br/backup-and-restore-tool/','/docs/dev/reference/tools/br/br/','/docs/dev/how-to/maintain/backup-and-restore/br/','/tidb/dev/backup-and-restore-tool/','/tidb/dev/point-in-time-recovery/'] +summary: TiDB Backup & Restore (BR) ensures high availability of clusters and data safety. It supports disaster recovery with a short RPO, handles misoperations, and provides history data auditing. It is recommended to perform backup operations during off-peak hours and store backup data to compatible storage systems. BR supports full backup and log backup, as well as restoring data to any point in time. It is important to use BR of the same major version as the TiDB cluster for backup and restoration. --- # TiDB Backup & Restore Overview @@ -17,7 +16,7 @@ BR satisfies the following requirements: ## Before you use -This section describes the prerequisites for using TiDB backup and restore, including restrictions, usage tips and compatibility issues. +This section describes the prerequisites for using TiDB backup and restore, including restrictions, usage tips and compatibility issues. For more information about the compatibility of the BR tool with other features or versions, see [Compatibility](#compatibility). ### Restrictions @@ -25,7 +24,7 @@ This section describes the prerequisites for using TiDB backup and restore, incl - PITR only supports cluster-level restore and does not support database-level or table-level restore. - PITR does not support restoring the data of user tables or privilege tables from system tables. - BR does not support running multiple backup tasks on a cluster **at the same time**. -- When a PITR is running, you cannot run a log backup task or use TiCDC to replicate data to a downstream cluster. +- When restoring a cluster using PITR, you cannot run a log backup task or use TiCDC to replicate data to a downstream cluster. ### Some tips @@ -94,7 +93,7 @@ Corresponding to the backup features, you can perform two types of restore: full #### Restore performance and impact on TiDB clusters - Data restore is performed at a scalable speed. Generally, the speed is 100 MiB/s per TiKV node. `br` only supports restoring data to a new cluster and uses the resources of the target cluster as much as possible. For more details, see [Restore performance and impact](/br/br-snapshot-guide.md#performance-and-impact-of-snapshot-restore). -- On each TiKV node, PITR can restore log data at 30 GiB/h. For more details, see [PITR performance and impact](/br/br-pitr-guide.md#performance-and-impact-of-pitr). +- On each TiKV node, PITR can restore log data at 30 GiB/h. For more details, see [PITR performance and impact](/br/br-pitr-guide.md#performance-capabilities-of-pitr). ## Backup storage @@ -113,7 +112,7 @@ Backup and restore might go wrong when some TiDB features are enabled or disable | ---- | ---- | ----- | |GBK charset|| BR of versions earlier than v5.4.0 does not support restoring `charset=GBK` tables. No version of BR supports recovering `charset=GBK` tables to TiDB clusters earlier than v5.4.0. | | Clustered index | [#565](https://github.com/pingcap/br/issues/565) | Make sure that the value of the `tidb_enable_clustered_index` global variable during restore is consistent with that during backup. Otherwise, data inconsistency might occur, such as `default not found` error and inconsistent data index. | -| New collation | [#352](https://github.com/pingcap/br/issues/352) | Make sure that the value of the `new_collations_enabled_on_first_bootstrap` variable during restore is consistent with that during backup. Otherwise, inconsistent data index might occur and checksum might fail to pass. For more information, see [FAQ - Why does BR report `new_collations_enabled_on_first_bootstrap` mismatch?](/faq/backup-and-restore-faq.md#why-is-new_collations_enabled_on_first_bootstrap-mismatch-reported-during-restore). | +| New collation | [#352](https://github.com/pingcap/br/issues/352) | Make sure that the value of the `new_collation_enabled` variable in the `mysql.tidb` table during restore is consistent with that during backup. Otherwise, inconsistent data index might occur and checksum might fail to pass. For more information, see [FAQ - Why does BR report `new_collations_enabled_on_first_bootstrap` mismatch?](/faq/backup-and-restore-faq.md#why-is-new_collation_enabled-mismatch-reported-during-restore). | | Global temporary tables | | Make sure that you are using v5.3.0 or a later version of BR to back up and restore data. Otherwise, an error occurs in the definition of the backed global temporary tables. | | TiDB Lightning Physical Import| | If the upstream database uses the physical import mode of TiDB Lightning, data cannot be backed up in log backup. It is recommended to perform a full backup after the data import. For more information, see [When the upstream database imports data using TiDB Lightning in the physical import mode, the log backup feature becomes unavailable. Why?](/faq/backup-and-restore-faq.md#when-the-upstream-database-imports-data-using-tidb-lightning-in-the-physical-import-mode-the-log-backup-feature-becomes-unavailable-why).| @@ -127,6 +126,8 @@ Before performing backup and restore, BR compares the TiDB cluster version with Starting from v7.0.0, TiDB gradually supports performing backup and restore operations through SQL statements. Therefore, it is strongly recommended to use the BR tool of the same major version as the TiDB cluster when backing up and restoring cluster data, and avoid performing data backup and restore operations across major versions. This helps ensure smooth execution of restore operations and data consistency. +#### BR version compatibility matrix before TiDB v6.6.0 + The compatibility information for BR before TiDB v6.6.0 is as follows: | Backup version (vertical) \ Restore version (horizontal) | Restore to TiDB v6.0 | Restore to TiDB v6.1 | Restore to TiDB v6.2 | Restore to TiDB v6.3, v6.4, or v6.5 | Restore to TiDB v6.6 | @@ -134,6 +135,36 @@ The compatibility information for BR before TiDB v6.6.0 is as follows: | TiDB v6.0, v6.1, v6.2, v6.3, v6.4, or v6.5 snapshot backup | Compatible (known issue [#36379](https://github.com/pingcap/tidb/issues/36379): if backup data contains an empty schema, BR might report an error.) | Compatible | Compatible | Compatible | Compatible (BR must be v6.6) | | TiDB v6.3, v6.4, v6.5, or v6.6 log backup| Incompatible | Incompatible | Incompatible | Compatible | Compatible | +#### BR version compatibility matrix between TiDB v6.5.0 and v7.5.0 + +This section introduces the BR compatibility information for all [Long-Term Support (LTS)](/releases/versioning.md#long-term-support-releases) versions between TiDB v6.5.0 and v7.5.0 (including v6.5.0, v7.1.0, v7.5.0): + +> **Note:** +> +> Known issue: Starting from version v7.2.0, some system table fields in newly created clusters are case-insensitive. However, for clusters that are **upgraded online** from versions earlier than v7.2.0 to v7.2.0 or later, the corresponding system table fields remain case-sensitive. Backup and restore operations involving system tables between these two types of clusters might fail. For more details, see [Issue #43717](https://github.com/pingcap/tidb/issues/43717). + +The following table lists the compatibility matrix for full backups. Note that all information in the table applies to newly created clusters. For clusters upgraded from a version earlier than v7.2.0 to v7.2.0 or later, their behavior is consistent with that of backups from v7.1.0. + +| Backup version | Compatible restore versions | Incompatible restore versions | +|:--|:--|:--| +| v6.5.0 | 7.1.0 | v7.5.0 and later | +| v7.1.0 | - | v7.5.0 and later | +| v7.5.0 | v7.5.0 and later | - | + +The following table lists the compatibility matrix for log backups. Note that all information in the table applies to newly created clusters. For clusters upgraded from a version earlier than v7.2.0 to v7.2.0 or later, their behavior is consistent with that of backups from v7.1.0. + +| Backup version | Compatible restore versions | Incompatible restore versions | +|:--|:--|:--| +| v6.5.0 | 7.1.0 | v7.5.0 and later | +| v7.1.0 | - | v7.5.0 and later | +| v7.5.0 | v7.5.0 and later | - | + +> **Note:** +> +> - When only data of non-system tables is backed up (full backup or log backup), all versions are compatible with each other. +> - In scenarios where restoring the `mysql` system table is incompatible, you can resolve the problem by setting `--with-sys-table=false` to skip restoring all system tables, or use a more fine-grained filter to just skip incompatible system tables, for example: `--filter '*.*' --filter "__TiDB_BR_Temporary_*.*" --filter '!mysql.*' --filter 'mysql.bind_info' --filter 'mysql.user' --filter 'mysql.global_priv' --filter 'mysql.global_grants' --filter 'mysql.default_roles' --filter 'mysql.role_edges' --filter '!sys.*' --filter '!INFORMATION_SCHEMA.*' --filter '!PERFORMANCE_SCHEMA.*' --filter '!METRICS_SCHEMA.*' --filter '!INSPECTION_SCHEMA.*'`. +> - `-` means that there are no compatibility restrictions for the corresponding scenario. + ## See also - [TiDB Snapshot Backup and Restore Guide](/br/br-snapshot-guide.md) diff --git a/br/backup-and-restore-storages.md b/br/backup-and-restore-storages.md index 309a7ae9f0821..498f80f8cfa11 100644 --- a/br/backup-and-restore-storages.md +++ b/br/backup-and-restore-storages.md @@ -1,7 +1,6 @@ --- title: Backup Storages -summary: Describes the storage URI format used in TiDB backup and restore. -aliases: ['/docs/dev/br/backup-and-restore-storages/','/tidb/dev/backup-storage-S3/','/tidb/dev/backup-storage-azblob/','/tidb/dev/backup-storage-gcs/','/tidb/dev/external-storage/'] +summary: TiDB supports backup storage to Amazon S3, Google Cloud Storage, Azure Blob Storage, and NFS. You can specify the URI and authentication for different storage services. BR sends credentials to TiKV by default when using S3, GCS, or Azure Blob Storage. You can disable this for cloud environments. The URI format for each storage service is specified, along with authentication methods. Server-side encryption is supported for Amazon S3 and Azure Blob Storage. BR v6.3.0 also supports AWS S3 Object Lock. --- # Backup Storages @@ -32,59 +31,13 @@ BACKUP DATABASE * TO 's3://bucket-name/prefix' SEND_CREDENTIALS_TO_TIKV = FALSE; ### URI format description -This section describes the URI format of the storage services: +The URI format of the external storage service is as follows: ```shell [scheme]://[host]/[path]?[parameters] ``` - -
- -- `scheme`: `s3` -- `host`: `bucket name` -- `parameters`: - - - `access-key`: Specifies the access key. - - `secret-access-key`: Specifies the secret access key. - - `session-token`: Specifies the temporary session token. BR does not support this parameter yet. - - `use-accelerate-endpoint`: Specifies whether to use the accelerate endpoint on Amazon S3 (defaults to `false`). - - `endpoint`: Specifies the URL of custom endpoint for S3-compatible services (for example, ``). - - `force-path-style`: Use path style access rather than virtual hosted style access (defaults to `true`). - - `storage-class`: Specifies the storage class of the uploaded objects (for example, `STANDARD` or `STANDARD_IA`). - - `sse`: Specifies the server-side encryption algorithm used to encrypt the uploaded objects (value options: ``, `AES256`, or `aws:kms`). - - `sse-kms-key-id`: Specifies the KMS ID if `sse` is set to `aws:kms`. - - `acl`: Specifies the canned ACL of the uploaded objects (for example, `private` or `authenticated-read`). - - `role-arn`: When you need to access Amazon S3 data from a third party using a specified [IAM role](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html), you can specify the corresponding [Amazon Resource Name (ARN)](https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) of the IAM role with the `role-arn` URL query parameter, such as `arn:aws:iam::888888888888:role/my-role`. For more information about using an IAM role to access Amazon S3 data from a third party, see [AWS documentation](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_common-scenarios_third-party.html). - - `external-id`: When you access Amazon S3 data from a third party, you might need to specify a correct [external ID](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user_externalid.html) to assume [the IAM role](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html). In this case, you can use this `external-id` URL query parameter to specify the external ID and make sure that you can assume the IAM role. An external ID is an arbitrary string provided by the third party together with the IAM role ARN to access the Amazon S3 data. Providing an external ID is optional when assuming an IAM role, which means if the third party does not require an external ID for the IAM role, you can assume the IAM role and access the corresponding Amazon S3 data without providing this parameter. - -
-
- -- `scheme`: `gcs` or `gs` -- `host`: `bucket name` -- `parameters`: - - - `credentials-file`: Specifies the path to the credentials JSON file on the migration tool node. - - `storage-class`: Specifies the storage class of the uploaded objects (for example, `STANDARD` or `COLDLINE`) - - `predefined-acl`: Specifies the predefined ACL of the uploaded objects (for example, `private` or `project-private`) - -
-
- -- `scheme`: `azure` or `azblob` -- `host`: `container name` -- `parameters`: - - - `account-name`: Specifies the account name of the storage. - - `account-key`: Specifies the access key. - - `sas-token`: Specifies the shared access signature (SAS) token. - - `access-tier`: Specifies the access tier of the uploaded objects, for example, `Hot`, `Cool`, or `Archive`. The default value is the default access tier of the storage account. - - `encryption-scope`: Specifies the [encryption scope](https://learn.microsoft.com/en-us/azure/storage/blobs/encryption-scope-manage?tabs=powershell#upload-a-blob-with-an-encryption-scope) for server-side encryption. - - `encryption-key`: Specifies the [encryption key](https://learn.microsoft.com/en-us/azure/storage/blobs/encryption-customer-provided-keys) for server-side encryption, which uses the AES256 encryption algorithm. - -
-
+For more information about the URI format, see [URI Formats of External Storage Services](/external-storage-uri.md). ### URI examples @@ -153,11 +106,15 @@ When storing backup data in a cloud storage system, you need to configure authen Before backup, configure the following privileges to access the backup directory on S3. -- Minimum privileges for TiKV and Backup & Restore (BR) to access the backup directories during backup: `s3:ListBucket`, `s3:PutObject`, and `s3:AbortMultipartUpload` -- Minimum privileges for TiKV and BR to access the backup directories during restore: `s3:ListBucket`, `s3:GetObject`, and `s3:PutObject`. BR writes checkpoint information to the `./checkpoints` subdirectory under the backup directory. When restoring log backup data, BR writes the table ID mapping relationship of the restored cluster to the `./pitr_id_maps` subdirectory under the backup directory. +- Minimum privileges for TiKV and Backup & Restore (BR) to access the backup directories during backup: `s3:ListBucket`, `s3:GetObject`, `s3:DeleteObject`, `s3:PutObject`, and `s3:AbortMultipartUpload` +- Minimum privileges for TiKV and BR to access the backup directories during restore: `s3:ListBucket`, `s3:GetObject`, `s3:DeleteObject`, and `s3:PutObject`. BR writes checkpoint information to the `./checkpoints` subdirectory under the backup directory. When restoring log backup data, BR writes the table ID mapping relationship of the restored cluster to the `./pitr_id_maps` subdirectory under the backup directory. If you have not yet created a backup directory, refer to [Create a bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket in the specified region. If necessary, you can also create a folder in the bucket by referring to [Create a folder](https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-folders.html). +> **Note:** +> +> In 2024, AWS changed the default behavior, and newly created instances now only support IMDSv2 by default. For more details, see [Set IMDSv2 as default for all new instance launches in your account](https://aws.amazon.com/about-aws/whats-new/2024/03/set-imdsv2-default-new-instance-launches/). Therefore, starting from v7.5.7, BR supports obtaining IAM role permissions on Amazon EC2 instances with only IMDSv2 enabled. When using BR of an earlier version before v7.5.7, you need to configure the instance to support both IMDSv1 and IMDSv2. + It is recommended that you configure access to S3 using either of the following ways: - Method 1: Specify the access key diff --git a/br/backup-and-restore-use-cases.md b/br/backup-and-restore-use-cases.md index 3d3af0b9a852a..42b39218c8387 100644 --- a/br/backup-and-restore-use-cases.md +++ b/br/backup-and-restore-use-cases.md @@ -1,7 +1,6 @@ --- title: TiDB Backup and Restore Use Cases -summary: Learn the use cases of backing up and restoring data using br command-line tool. -aliases: ['/docs/dev/br/backup-and-restore-use-cases/','/docs/dev/reference/tools/br/use-cases/','/tidb/dev/backup-and-restore-use-cases-for-maintain/'] +summary: TiDB provides snapshot and log backup solutions for specific use cases, such as timely data recovery and business audits. To use point-in-time recovery (PITR), deploy a TiDB cluster >= v6.2.0 and update BR to v7.6.0. Configure backup storage on Amazon S3 and set a backup policy to meet data loss and recovery requirements. Run log and snapshot backups, and use PITR to restore data to a specific time point. Clean up outdated data regularly. For detailed steps, refer to TiDB documentation. --- # TiDB Backup and Restore Use Cases @@ -17,7 +16,7 @@ With PITR, you can satisfy the preceding requirements. ## Deploy the TiDB cluster and BR -To use PITR, you need to deploy a TiDB cluster >= v6.2.0 and update BR to the same version as the TiDB cluster. This document uses v7.4.0 as an example. +To use PITR, you need to deploy a TiDB cluster >= v6.2.0 and update BR to the same version as the TiDB cluster. This document uses v{{{ .tidb-version }}} as an example. The following table shows the recommended hardware resources for using PITR in a TiDB cluster. @@ -44,13 +43,13 @@ Install or upgrade BR using TiUP: - Install: ```shell - tiup install br:v7.4.0 + tiup install br:v{{{ .tidb-version }}} ``` - Upgrade: ```shell - tiup update br:v7.4.0 + tiup update br:v{{{ .tidb-version }}} ``` ## Configure backup storage (Amazon S3) @@ -70,8 +69,8 @@ The detailed steps are as follows: 2. Configure permissions for BR and TiKV to access the S3 directory. It is recommended to grant permissions using the IAM method, which is the most secure way to access the S3 bucket. For detailed steps, refer to [AWS documentation: Controlling access to a bucket with user policies](https://docs.aws.amazon.com/AmazonS3/latest/userguide/walkthrough1.html). The required permissions are as follows: - - TiKV and BR in the backup cluster need `s3:ListBucket`, `s3:PutObject`, and `s3:AbortMultipartUpload` permissions of the `s3://tidb-pitr-bucket/backup-data` directory. - - TiKV and BR in the restore cluster need `s3:ListBucket`, `s3:GetObject`, and `s3:PutObject` permissions of the `s3://tidb-pitr-bucket/backup-data` directory. + - TiKV and BR in the backup cluster need `s3:ListBucket`, `s3:GetObject`, `s3:DeleteObject`, `s3:PutObject`, and `s3:AbortMultipartUpload` permissions of the `s3://tidb-pitr-bucket/backup-data` directory. + - TiKV and BR in the restore cluster need `s3:ListBucket`, `s3:GetObject`, `s3:DeleteObject`, and `s3:PutObject` permissions of the `s3://tidb-pitr-bucket/backup-data` directory. 3. Plan the directory structure that stores the backup data, including the snapshot (full) backup and the log backup. @@ -122,7 +121,7 @@ The following are two snapshot backup examples: ```shell tiup br backup full --pd="${PD_IP}:2379" \ --storage='s3://tidb-pitr-bucket/backup-data/snapshot-20220514000000' \ - --backupts='2022/05/14 00:00:00' + --backupts='2022/05/14 00:00:00 +08:00' ``` - Run a snapshot backup at 2022/05/16 00:00:00 @@ -130,7 +129,7 @@ The following are two snapshot backup examples: ```shell tiup br backup full --pd="${PD_IP}:2379" \ --storage='s3://tidb-pitr-bucket/backup-data/snapshot-20220516000000' \ - --backupts='2022/05/16 00:00:00' + --backupts='2022/05/16 00:00:00 +08:00' ``` ## Run PITR diff --git a/br/br-auto-tune.md b/br/br-auto-tune.md index 643df204a9c25..49d50761556f7 100644 --- a/br/br-auto-tune.md +++ b/br/br-auto-tune.md @@ -1,6 +1,6 @@ --- title: Backup Auto-Tune -summary: Learn about the auto-tune feature of TiDB backup and restore, which automatically limits the resources used by backups to reduce the impact on the cluster in case of high cluster resource usage. +summary: TiDB v5.4.0 introduces the auto-tune feature for backup tasks, which is enabled by default. It limits the resources used by backup tasks to reduce their impact on the cluster. You can enable or disable the feature dynamically without restarting the cluster. However, auto-tune may not completely remove the impact of backup on the cluster due to limitations. Adjusting the number of threads used by backup tasks can help mitigate the impact in certain scenarios. --- # Backup Auto-Tune New in v5.4.0 @@ -13,7 +13,7 @@ To reduce the impact of backup tasks on the cluster, TiDB v5.4.0 introduces the If you want to reduce the impact of backup tasks on the cluster, you can enable the auto-tune feature. With this feature enabled, TiDB performs backup tasks as fast as possible without excessively affecting the cluster. -Alternatively, you can limit the backup speed by using the TiKV configuration item [`backup.num-threads`](/tikv-configuration-file.md#num-threads-1) or using the parameter `--ratelimit`. +Alternatively, you can limit the backup speed by using the TiKV configuration item [`backup.num-threads`](/tikv-configuration-file.md#num-threads-1) or using the parameter `--ratelimit`. When `--ratelimit` is set, to avoid too many tasks causing the speed limit to fail, the `concurrency` parameter of br is automatically adjusted to `1`. ## Use auto-tune @@ -30,7 +30,7 @@ TiKV supports dynamically configuring the auto-tune feature. You can enable or d {{< copyable "shell-regular" >}} ```shell -tikv-ctl modify-tikv-config -n backup.enable-auto-tune -v +tikv-ctl --host= modify-tikv-config -n backup.enable-auto-tune -v ``` When you perform backup tasks on an offline cluster, to speed up the backup, you can modify the value of `backup.num-threads` to a larger number using `tikv-ctl`. diff --git a/br/br-batch-create-table.md b/br/br-batch-create-table.md index 1f68e8c6d0c5b..842bad3b156ad 100644 --- a/br/br-batch-create-table.md +++ b/br/br-batch-create-table.md @@ -1,6 +1,6 @@ --- title: Batch Create Table -summary: Learn how to use the Batch Create Table feature. When restoring data, BR can create tables in batches to speed up the restore process. +summary: TiDB v6.0.0 introduces the Batch Create Table feature to speed up the table creation process during data restoration. It is enabled by default and creates tables in batches, significantly reducing the time for restoring data with a large number of tables. The feature test shows that the average speed of restoring one TiKV instance is as high as 181.65 MB/s. --- # Batch Create Table diff --git a/br/br-checkpoint-backup.md b/br/br-checkpoint-backup.md index f5dcd7a14f5f9..6f9ba3b68d674 100644 --- a/br/br-checkpoint-backup.md +++ b/br/br-checkpoint-backup.md @@ -1,7 +1,6 @@ --- title: Checkpoint Backup -summary: Learn about the checkpoint backup feature, including its application scenarios, implementation details, and usage. -aliases: ["/tidb/dev/br-checkpoint"] +summary: TiDB v6.5.0 introduces checkpoint backup feature to continue interrupted backups, reducing the need to start from scratch. It records backed up shards to resume backup progress, but relies on GC mechanism and may require some data to be backed up again. The `br` tool periodically updates `gc-safepoint` to avoid data being garbage collected, and can extend retention period if needed. --- # Checkpoint Backup diff --git a/br/br-checkpoint-restore.md b/br/br-checkpoint-restore.md index 79276c61d6480..0a6c453ade6ae 100644 --- a/br/br-checkpoint-restore.md +++ b/br/br-checkpoint-restore.md @@ -1,6 +1,6 @@ --- title: Checkpoint Restore -summary: Learn about the checkpoint restore feature, including its application scenarios, implementation details, and usage. +summary: TiDB v7.1.0 introduces checkpoint restore, allowing interrupted snapshot and log restores to continue without starting from scratch. It records restored shards and table IDs, enabling retries to use the progress point close to the interruption. However, it relies on the GC mechanism and may require some data to be restored again. It's important to avoid modifying cluster data during the restore to ensure accuracy. --- # Checkpoint Restore diff --git a/br/br-incremental-guide.md b/br/br-incremental-guide.md index 03e29d2f893d6..e0c4c552e73c0 100644 --- a/br/br-incremental-guide.md +++ b/br/br-incremental-guide.md @@ -1,6 +1,6 @@ --- title: TiDB Incremental Backup and Restore Guide -summary: Learns about how to perform incremental backup and restore in TiDB. +summary: Incremental data is the differentiated data between starting and end snapshots, along with DDLs. It reduces backup volume and requires setting `tidb_gc_life_time` for incremental backup. Use `br backup` with `--lastbackupts` for incremental backup and ensure all previous data is restored before restoring incremental data. --- # TiDB Incremental Backup and Restore Guide @@ -11,6 +11,12 @@ Incremental data of a TiDB cluster is differentiated data between the starting s > > Development for this feature has stopped. It is recommended that you use [log backup and PITR](/br/br-pitr-guide.md) as an alternative. +## Limitations + +Because restoring the incremental backup relies on the snapshot of the database table at the backup time point to filter incremental DDL statements, the tables deleted during the incremental backup process might still exist after the data restore and need to be manually deleted. + +The incremental backup does not support batch renaming of tables. If batch renaming of tables occurs during the incremental backup process, the data restore might fail. It is recommended to perform a full backup after batch renaming tables, and use the latest full backup to replace the incremental data during restore. + ## Back up incremental data To back up incremental data, run the `br backup` command with **the last backup timestamp** `--lastbackupts` specified. In this way, br command-line tool automatically backs up incremental data generated between `lastbackupts` and the current time. To get `--lastbackupts`, run the `validate` command. The following is an example: diff --git a/br/br-log-architecture.md b/br/br-log-architecture.md index c75f3593dc75c..d27d4585ca5c7 100644 --- a/br/br-log-architecture.md +++ b/br/br-log-architecture.md @@ -1,6 +1,6 @@ --- title: TiDB Log Backup and PITR Architecture -summary: Learn about the architecture of TiDB log backup and point-in-time recovery. +summary: TiDB log backup and PITR architecture is introduced using a Backup & Restore (BR) tool as an example. The architecture includes log backup process design, system components, and key concepts. The PITR process involves restoring full backup data and log backup data. Log backup generates files such as log data, metadata, and global checkpoint. --- # TiDB Log Backup and PITR Architecture @@ -97,9 +97,9 @@ The complete PITR process is as follows: Log backup generates the following types of files: +- `{resolved_ts}-{uuid}.meta` file: is generated every time each TiKV node uploads the log backup data and stores metadata of all log backup data files uploaded this time. The `{resolved_ts}` is the resolved timestamp of the TiKV node. The latest `resolved_ts` of the log backup task is the minimum resolved timestamp among all TiKV nodes. The `{uuid}` is generated randomly when the file is created. +- `{store_id}.ts` file: is updated with global checkpoint ts every time each TiKV node uploads the log backup data. The `{store_id}` is the store ID of the TiKV node. - `{min_ts}-{uuid}.log` file: stores the KV change log data of the backup task. The `{min_ts}` is the minimum TSO timestamp of the KV change log data in the file, and the `{uuid}` is generated randomly when the file is created. -- `{checkpoint_ts}-{uuid}.meta` file: is generated every time each TiKV node uploads the log backup data and stores metadata of all log backup data files uploaded this time. The `{checkpoint_ts}` is the log backup checkpoint of the TiKV node, and the global checkpoint is the minimum checkpoint of all TiKV nodes. The `{uuid}` is generated randomly when the file is created. -- `{store_id}.ts` file: this file is updated with global checkpoint ts every time each TiKV node uploads the log backup data. The `{store_id}` is the store ID of the TiKV node. - `v1_stream_truncate_safepoint.txt` file: stores the timestamp corresponding to the latest backup data in storage that deleted by `br log truncate`. ### Structure of backup files @@ -108,18 +108,24 @@ Log backup generates the following types of files: . ├── v1 │   ├── backupmeta -│   │   ├── {min_restored_ts}-{uuid}.meta -│   │   ├── {checkpoint}-{uuid}.meta +│   │   ├── ... +│   │   └── {resolved_ts}-{uuid}.meta │   ├── global_checkpoint -│   │   ├── {store_id}.ts -│   ├── {date} -│   │   ├── {hour} -│   │   │   ├── {store_id} -│   │   │   │   ├── {min_ts}-{uuid}.log -│   │   │   │   ├── {min_ts}-{uuid}.log -├── v1_stream_truncate_safepoint.txt +│   │   └── {store_id}.ts +│   └── {date} +│      └── {hour} +│         └── {store_id} +│            ├── ... +│            └── {min_ts}-{uuid}.log +└── v1_stream_truncate_safepoint.txt ``` +Explanation of the backup file directory structure: + +- `backupmeta`: stores backup metadata. The `resolved_ts` in the filename indicates the backup progress, meaning that data before this TSO has been fully backed up. However, note that this TSO only reflects the progress of certain shards. +- `global_checkpoint`: represents the global backup progress. It records the latest point in time to which data can be restored using `br restore point`. +- `{date}/{hour}`: stores backup data for the corresponding date and hour. When cleaning up storage, always use `br log truncate` instead of manually deleting data. This is because the metadata references the data in this directory, and manual deletion might lead to restore failures or data inconsistencies after restore. + The following is an example: ``` @@ -129,24 +135,24 @@ The following is an example: │   │   ├── ... │   │   ├── 435213818858112001-e2569bda-a75a-4411-88de-f469b49d6256.meta │   │   ├── 435214043785779202-1780f291-3b8a-455e-a31d-8a1302c43ead.meta -│   │   ├── 435214443785779202-224f1408-fff5-445f-8e41-ca4fcfbd2a67.meta +│   │   └── 435214443785779202-224f1408-fff5-445f-8e41-ca4fcfbd2a67.meta │   ├── global_checkpoint │   │   ├── 1.ts │   │   ├── 2.ts -│   │   ├── 3.ts -│   ├── 20220811 -│   │   ├── 03 -│   │   │   ├── 1 -│   │   │   │   ├── ... -│   │   │   │   ├── 435213866703257604-60fcbdb6-8f55-4098-b3e7-2ce604dafe54.log -│   │   │   │   ├── 435214023989657606-72ce65ff-1fa8-4705-9fd9-cb4a1e803a56.log -│   │   │   ├── 2 -│   │   │   │   ├── ... -│   │   │   │   ├── 435214102632857605-11deba64-beff-4414-bc9c-7a161b6fb22c.log -│   │   │   │   ├── 435214417205657604-e6980303-cbaa-4629-a863-1e745d7b8aed.log -│   │   │   ├── 3 -│   │   │   │   ├── ... -│   │   │   │   ├── 435214495848857605-7bf65e92-8c43-427e-b81e-f0050bd40be0.log -│   │   │   │   ├── 435214574492057604-80d3b15e-3d9f-4b0c-b133-87ed3f6b2697.log -├── v1_stream_truncate_safepoint.txt +│   │   └── 3.ts +│   └── 20220811 +│      └── 03 +│         ├── 1 +│         │   ├── ... +│         │   ├── 435213866703257604-60fcbdb6-8f55-4098-b3e7-2ce604dafe54.log +│         │   └── 435214023989657606-72ce65ff-1fa8-4705-9fd9-cb4a1e803a56.log +│         ├── 2 +│         │   ├── ... +│         │   ├── 435214102632857605-11deba64-beff-4414-bc9c-7a161b6fb22c.log +│         │   └── 435214417205657604-e6980303-cbaa-4629-a863-1e745d7b8aed.log +│         └── 3 +│            ├── ... +│            ├── 435214495848857605-7bf65e92-8c43-427e-b81e-f0050bd40be0.log +│            └── 435214574492057604-80d3b15e-3d9f-4b0c-b133-87ed3f6b2697.log +└── v1_stream_truncate_safepoint.txt ``` diff --git a/br/br-monitoring-and-alert.md b/br/br-monitoring-and-alert.md index ca3cb2fdfbcf9..f9bc07035aad6 100644 --- a/br/br-monitoring-and-alert.md +++ b/br/br-monitoring-and-alert.md @@ -1,12 +1,16 @@ --- title: Monitoring and Alert for Backup and Restore -summary: Learn the monitoring and alert of the backup and restore feature. +summary: This document describes monitoring and alert for backup and restore, including log backup monitoring, configuration, Grafana configuration, monitoring metrics, and log backup alerts. It covers the recommended alert items and their configurations for PITR. --- # Monitoring and Alert for Backup and Restore This document describes the monitoring and alert of the backup and restore feature, including how to deploy monitoring components, monitoring metrics, and common alerts. +## Snapshot backup and restore monitoring + +To view the snapshot backup and restore metrics, go to the [**TiKV-Details** > **Backup & Import** dashboard](/grafana-tikv-dashboard.md#backup--import) in Grafana. + ## Log backup monitoring Log backup supports using [Prometheus](https://prometheus.io/) to collect monitoring metrics. Currently all monitoring metrics are built into TiKV. @@ -19,13 +23,13 @@ Log backup supports using [Prometheus](https://prometheus.io/) to collect monito ### Grafana configuration - For clusters deployed using TiUP, the [Grafana](https://grafana.com/) dashboard contains the point-in-time recovery (PITR) panel. The **Backup Log** panel in the TiKV-Details dashboard is the PITR panel. -- For clusters deployed manually, refer to [Import a Grafana dashboard](/deploy-monitoring-services.md#step-2-import-a-grafana-dashboard) and upload the [tikv_details](https://github.com/tikv/tikv/blob/master/metrics/grafana/tikv_details.json) JSON file to Grafana. Then find the **Backup Log** panel in the TiKV-Details dashboard. +- For clusters deployed manually, refer to [Import a Grafana dashboard](/deploy-monitoring-services.md#step-2-import-a-grafana-dashboard) and upload the [tikv_details](https://github.com/tikv/tikv/blob/release-7.5/metrics/grafana/tikv_details.json) JSON file to Grafana. Then find the **Backup Log** panel in the TiKV-Details dashboard. ### Monitoring metrics | Metrics | Type | Description | |-------------------------------------------------------|-----------|---------------------------------------------------------------------------------------------------------------------------------------------------------| -| **tikv_log_backup_interal_actor_acting_duration_sec** | Histogram | The duration of handling all internal messages and events.
`message :: TaskType` | +| **tikv_log_backup_internal_actor_acting_duration_sec** | Histogram | The duration of handling all internal messages and events.
`message :: TaskType` | | **tikv_log_backup_initial_scan_reason** | Counter | Statistics of the reasons why initial scan is triggered. The main reason is leader transfer or Region version change.
`reason :: {"leader-changed", "region-changed", "retry"}` | | **tikv_log_backup_event_handle_duration_sec** | Histogram | The duration of handling KV events. Compared with `tikv_log_backup_on_event_duration_seconds`, this metric also includes the duration of internal conversion.
`stage :: {"to_stream_event", "save_to_temp_file"}` | | **tikv_log_backup_handle_kv_batch** | Histogram | Region-level statistics of the sizes of KV pair batches sent by Raftstore. | diff --git a/br/br-pitr-guide.md b/br/br-pitr-guide.md index 00b0b5d8b23f9..1e81d8395b932 100644 --- a/br/br-pitr-guide.md +++ b/br/br-pitr-guide.md @@ -1,7 +1,6 @@ --- title: TiDB Log Backup and PITR Guide -summary: Learns about how to perform log backup and PITR in TiDB. -aliases: ['/tidb/dev/pitr-usage'] +summary: TiDB Log Backup and PITR Guide explains how to back up and restore data using the br command-line tool. It includes instructions for starting log backup, running full backup regularly, and cleaning up outdated data. The guide also provides information on running PITR and the performance capabilities of PITR. --- # TiDB Log Backup and PITR Guide @@ -23,7 +22,7 @@ To start a log backup, run `br log start`. A cluster can only run one log backup ```shell tiup br log start --task-name=pitr --pd "${PD_IP}:2379" \ ---storage 's3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}"' +--storage 's3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}' ``` After the log backup task starts, it runs in the background of the TiDB cluster until you stop it manually. During this process, the TiDB change logs are regularly backed up to the specified storage in small batches. To query the status of the log backup task, run the following command: @@ -52,7 +51,7 @@ The snapshot backup can be used as a method of full backup. You can run `br back ```shell tiup br backup full --pd "${PD_IP}:2379" \ ---storage 's3://backup-101/snapshot-${date}?access-key=${access-key}&secret-access-key=${secret-access-key}"' +--storage 's3://backup-101/snapshot-${date}?access-key=${access-key}&secret-access-key=${secret-access-key}' ``` ## Run PITR @@ -61,8 +60,8 @@ To restore the cluster to any point in time within the backup retention period, ```shell br restore point --pd "${PD_IP}:2379" \ ---storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}"' \ ---full-backup-storage='s3://backup-101/snapshot-${date}?access-key=${access-key}&secret-access-key=${secret-access-key}"' \ +--storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}' \ +--full-backup-storage='s3://backup-101/snapshot-${date}?access-key=${access-key}&secret-access-key=${secret-access-key}' \ --restored-ts '2022-05-15 18:00:00+0800' ``` @@ -94,7 +93,7 @@ The following steps describe how to clean up backup data that exceeds the backup 3. Delete log backup data earlier than the snapshot backup `FULL_BACKUP_TS`: ```shell - tiup br log truncate --until=${FULL_BACKUP_TS} --storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}"' + tiup br log truncate --until=${FULL_BACKUP_TS} --storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}' ``` 4. Delete snapshot data earlier than the snapshot backup `FULL_BACKUP_TS`: @@ -103,12 +102,10 @@ The following steps describe how to clean up backup data that exceeds the backup rm -rf s3://backup-101/snapshot-${date} ``` -## Performance and impact of PITR +## Performance capabilities of PITR -### Capabilities - -- On each TiKV node, PITR can restore snapshot data at a speed of 280 GB/h and log data 30 GB/h. -- BR deletes outdated log backup data at a speed of 600 GB/h. +- On each TiKV node, PITR can restore snapshot data (full restore) at a speed of 280 GB/h and log data (including meta files and KV files) at a speed of 30 GB/h. +- BR deletes outdated log backup data (`tiup br log truncate`) at a speed of 600 GB/h. > **Note:** > @@ -119,9 +116,15 @@ The following steps describe how to clean up backup data that exceeds the backup > > The snapshot data size refers to the logical size of all KVs in a single replica, not the actual amount of restored data. BR restores all replicas according to the number of replicas configured for the cluster. The more replicas there are, the more data can be actually restored. > The default replica number for all clusters in the test is 3. -> To improve the overall restore performance, you can modify the [`import.num-threads`](/tikv-configuration-file.md#import) item in the TiKV configuration file and the [`concurrency`](/br/use-br-command-line-tool.md#common-options) option in the BR command. +> To improve the overall restore performance, you can modify the [`import.num-threads`](/tikv-configuration-file.md#import) item in the TiKV configuration file and the [`pitr-concurrency`](/br/br-pitr-manual.md#restore-to-a-specified-point-in-time-pitr) option in the BR command. +> When the upstream cluster has **many Regions** and a **short flush interval**, PITR generates a large number of small files. This increases batching and dispatching overhead during restore. To raise the number of files processed per batch, you can **moderately** increase the values of the following parameters: +> +> - `pitr-batch-size`: cumulative **bytes per batch** (default **16 MiB**). +> - `pitr-batch-count`: **number of files per batch** (default **8**). +> +> When determining whether to start the next batch, these two thresholds are evaluated independently: whichever threshold is reached first closes the current batch and starts the next, while the other threshold is ignored for that batch. -Testing scenario 1 (on [TiDB Cloud](https://tidbcloud.com)): +Testing scenario 1 (on [TiDB Cloud](https://tidbcloud.com)) is as follows: - The number of TiKV nodes (8 core, 16 GB memory): 21 - TiKV configuration item `import.num-threads`: 8 @@ -130,7 +133,7 @@ Testing scenario 1 (on [TiDB Cloud](https://tidbcloud.com)): - New log data created in the cluster: 10 GB/h - Write (INSERT/UPDATE/DELETE) QPS: 10,000 -Testing scenario 2 (on TiDB Self-Hosted): +Testing scenario 2 (on TiDB Self-Managed) is as follows: - The number of TiKV nodes (8 core, 64 GB memory): 6 - TiKV configuration item `import.num-threads`: 8 @@ -139,8 +142,15 @@ Testing scenario 2 (on TiDB Self-Hosted): - New log data created in the cluster: 10 GB/h - Write (INSERT/UPDATE/DELETE) QPS: 10,000 +## Monitoring and alert + +After log backup tasks are distributed, each TiKV node continuously writes data to external storage. You can view the monitoring data for this process in the **TiKV-Details > Backup Log** dashboard. + +To receive notifications metrics deviate from normal ranges, see [Log backup alerts](/br/br-monitoring-and-alert.md#log-backup-alerts) to configure alert rules. + ## See also * [TiDB Backup and Restore Use Cases](/br/backup-and-restore-use-cases.md) * [br Command-line Manual](/br/use-br-command-line-tool.md) * [Log Backup and PITR Architecture](/br/br-log-architecture.md) +* [Monitoring and Alert for Backup and Restore](/br/br-monitoring-and-alert.md) diff --git a/br/br-pitr-manual.md b/br/br-pitr-manual.md index 1c2095d24620a..49678f7e0ec2e 100644 --- a/br/br-pitr-manual.md +++ b/br/br-pitr-manual.md @@ -1,7 +1,6 @@ --- title: TiDB Log Backup and PITR Command Manual -summary: Learn about the commands of TiDB log backup and point-in-time recovery. -aliases: ['/tidb/dev/br-log-command-line/'] +summary: TiDB Log Backup and PITR Command Manual describes commands for log backup and point-in-time recovery. Use `br log` command to start, pause, resume, stop, truncate, and query log backup tasks. Specify parameters like `start-ts`, `task-name`, `--storage`, and `--pd` for log backup. Use `br log metadata` to view backup metadata and `br restore point` for PITR. Be cautious when pausing and resuming backup tasks. --- # TiDB Log Backup and PITR Command Manual @@ -290,7 +289,7 @@ Usage example: ```shell ./br log truncate --until='2022-07-26 21:20:00+0800' \ -–-storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}"' +--storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}' ``` Expected output: @@ -330,7 +329,7 @@ The `--storage` parameter is used to specify the backup storage address. Current Usage example: ```shell -./br log metadata –-storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}"' +./br log metadata --storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}' ``` Expected output: @@ -355,6 +354,9 @@ Usage: Flags: --full-backup-storage string specify the backup full storage. fill it if want restore full backup before restore log. -h, --help help for point + --pitr-batch-count uint32 specify the batch count to restore log. (default 8) + --pitr-batch-size uint32 specify the batch size to restore log. (default 16777216) + --pitr-concurrency uint32 specify the concurrency to restore log. (default 16) --restored-ts string the point of restore, used for log restore. support TSO or datetime, e.g. '400036290571534337' or '2018-05-11 01:42:23+0800' --start-ts string the start timestamp which log restore from. support TSO or datetime, e.g. '400036290571534337' or '2018-05-11 01:42:23+0800' @@ -369,7 +371,10 @@ Global Flags: The example output only shows the common parameters. These parameters are described as follows: -- `--full-backup-storage`: the storage address for the snapshot (full) backup. To use PITR, specify this parameter and choose the latest snapshot backup before the restore timestamp. To restore only log backup data, you can omit this parameter. Currently, BR supports Amazon S3, GCS, or Azure Blob Storage as the storage for log backup. For details, see [URI Formats of External Storage Services](/external-storage-uri.md). +- `--full-backup-storage`: the storage address for the snapshot (full) backup. To use PITR, specify this parameter and choose the latest snapshot backup before the restore timestamp. To restore only log backup data, you can omit this parameter. Note that when initializing the recovery cluster for the first time, you must specify a snapshot backup. Currently, BR supports Amazon S3, GCS, and Azure Blob Storage as the storage for log backup. For details, see [URI Formats of External Storage Services](/external-storage-uri.md). +- `--pitr-batch-count`: the maximum number of files in a single batch when restoring log data. Once this threshold is reached, the current batch ends immediately and the next batch starts. +- `--pitr-batch-size`: the maximum data size (in bytes) in a single batch when restoring log data. Once this threshold is reached, the current batch ends immediately and the next batch starts. +- `--pitr-concurrency`: the number of concurrent tasks during log restore. Each concurrent task restores one batch of log data at a time. - `--restored-ts`: the timestamp that you want to restore data to. If this parameter is not specified, BR restores data to the latest timestamp available in the log backup, that is, the checkpoint of the backup data. - `--start-ts`: the start timestamp that you want to restore log backup data from. If you only need to restore log backup data, you must specify this parameter. - `--pd`: the PD address of the restore cluster. @@ -381,7 +386,7 @@ Usage example: ```shell ./br restore point --pd="${PD_IP}:2379" --storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}"' ---full-backup-storage='s3://backup-101/snapshot-202205120000?access-key=${access-key}&secret-access-key=${secret-access-key}"' +--full-backup-storage='s3://backup-101/snapshot-202205120000?access-key=${access-key}&secret-access-key=${secret-access-key}' Full Restore <--------------------------------------------------------------------------------------------------------------------------------------------------------> 100.00% *** ***["Full Restore success summary"] ****** [total-take=3.112928252s] [restore-data-size(after-compressed)=5.056kB] [Size=5056] [BackupTS=434693927394607136] [total-kv=4] [total-kv-size=290B] [average-speed=93.16B/s] @@ -392,5 +397,6 @@ Restore KV Files <-------------------------------------------------------------- > **Note:** > +> - When you restore the cluster for the first time, you must specify the full snapshot data. Otherwise, some data in the newly created table might be incorrect due to rewriting Table ID rules. > - You cannot restore the log backup data of a certain time period repeatedly. If you restore the log backup data of a range `[t1=10, t2=20)` repeatedly, the restored data might be inconsistent. > - When you restore log data of different time periods in multiple batches, ensure that the log data is restored in consecutive order. If you restore the log backup data of `[t1, t2)`, `[t2, t3)`, and `[t3, t4)` in consecutive order, the restored data is consistent. However, if you restore `[t1, t2)` and then skip `[t2, t3)` to restore `[t3, t4)`, the restored data might be inconsistent. diff --git a/br/br-snapshot-architecture.md b/br/br-snapshot-architecture.md index bfedcc8c039d8..5af26ce3112cf 100644 --- a/br/br-snapshot-architecture.md +++ b/br/br-snapshot-architecture.md @@ -1,6 +1,6 @@ --- title: TiDB Snapshot Backup and Restore Architecture -summary: Learn about the architecture of TiDB snapshot backup and restore. +summary: TiDB Snapshot Backup and Restore Architecture introduces the process using a Backup & Restore (BR) tool. The architecture includes backup and restore processes, types of backup files, naming format, storage format, and structure of backup files. The backup process involves scheduling, data backup, and metadata backup. The restore process includes scheduling, schema restore, Region allocation, data restore, and reporting. The types of backup files include SST, backupmeta, and backup.lock files. The naming format and storage format of SST files are explained in detail. For more information, refer to the TiDB snapshot backup and restore guide. --- # TiDB Snapshot Backup and Restore Architecture diff --git a/br/br-snapshot-guide.md b/br/br-snapshot-guide.md index 3827c356dc36e..ffe4b0f1779c4 100644 --- a/br/br-snapshot-guide.md +++ b/br/br-snapshot-guide.md @@ -1,7 +1,6 @@ --- title: Snapshot Backup and Restore Guide -summary: Learn about how to back up and restore TiDB snapshots using the br command-line tool. -aliases: ['/tidb/dev/br-usage-backup/','/tidb/dev/br-usage-restore/','/tidb/dev/br-usage-restore-for-maintain/', '/tidb/dev/br-usage-backup-for-maintain/'] +summary: This document describes how to back up and restore TiDB snapshots using the br command-line tool. It includes instructions for snapshot backup, restoring data of a specified time point, and restoring a database or table. The document also covers the performance and impact of snapshot backup and restore. --- # Snapshot Backup and Restore Guide @@ -26,14 +25,14 @@ You can back up a TiDB cluster snapshot by running the `br backup full` command. ```shell tiup br backup full --pd "${PD_IP}:2379" \ - --backupts '2022-09-08 13:30:00' \ + --backupts '2022-09-08 13:30:00 +08:00' \ --storage "s3://backup-101/snapshot-202209081330?access-key=${access-key}&secret-access-key=${secret-access-key}" \ --ratelimit 128 \ ``` In the preceding command: -- `--backupts`: The time point of the snapshot. The format can be [TSO](/glossary.md#tso) or timestamp, such as `400036290571534337` or `2018-05-11 01:42:23`. If the data of this snapshot is garbage collected, the `br backup` command returns an error and `br` exits. If you leave this parameter unspecified, `br` picks the snapshot corresponding to the backup start time. +- `--backupts`: The time point of the snapshot. The format can be [TSO](/glossary.md#tso) or timestamp, such as `400036290571534337` or `2018-05-11 01:42:23 +08:00`. If the data of this snapshot is garbage collected, the `br backup` command returns an error and `br` exits. When backing up using a timestamp, it is recommended to specify the time zone as well. Otherwise, `br` uses the local time zone to construct the timestamp by default, which might lead to an incorrect backup time point. If you leave this parameter unspecified, `br` picks the snapshot corresponding to the backup start time. - `--storage`: The storage address of the backup data. Snapshot backup supports Amazon S3, Google Cloud Storage, and Azure Blob Storage as backup storage. The preceding command uses Amazon S3 as an example. For more details, see [URI Formats of External Storage Services](/external-storage-uri.md). - `--ratelimit`: The maximum speed **per TiKV** performing backup tasks. The unit is in MiB/s. @@ -143,7 +142,7 @@ Starting from BR v5.1.0, when you back up snapshots, BR backs up the **system ta - Statistics tables (`mysql.stat_*`). But statistics can be restored. See [Back up statistics](/br/br-snapshot-manual.md#back-up-statistics). - System variable tables (`mysql.tidb` and `mysql.global_variables`) -- [Other system tables](https://github.com/pingcap/tidb/blob/master/br/pkg/restore/systable_restore.go#L31) +- [Other system tables](https://github.com/pingcap/tidb/blob/release-7.5/br/pkg/restore/systable_restore.go#L31) ``` +-----------------------------------------------------+ @@ -167,14 +166,11 @@ Starting from BR v5.1.0, when you back up snapshots, BR backs up the **system ta +-----------------------------------------------------+ ``` -When you restore data related to system privilege, note the following: +When you restore data related to system privilege, note that before restoring data, BR checks whether the system tables in the target cluster are compatible with those in the backup data. "Compatible" means that all the following conditions are met: -- BR does not restore user data with `user` as `cloud_admin` and `host` as `'%'`. This user is reserved for TiDB Cloud. Do not create a user or role named `cloud_admin` in your cluster, because the user privileges related to `cloud_admin` cannot be restored correctly. -- Before restoring data, BR checks whether the system tables in the target cluster are compatible with those in the backup data. "Compatible" means that all the following conditions are met: - - - The target cluster has the same system tables as the backup data. - - The **number of columns** in the system privilege table of the target cluster is the same as that in the backup data. The column order is not important. - - The columns in the system privilege table of the target cluster are compatible with that in the backup data. If the data type of the column is a type with a length (such as integer and string), the length in the target cluster must be >= the length in the backup data. If the data type of the column is an `ENUM` type, the number of `ENUM` values in the target cluster must be a superset of that in the backup data. +- The target cluster has the same system tables as the backup data. +- The **number of columns** in the system privilege table of the target cluster is the same as that in the backup data. The column order is not important. +- The columns in the system privilege table of the target cluster are compatible with that in the backup data. If the data type of the column is a type with a length (such as integer and string), the length in the target cluster must be >= the length in the backup data. If the data type of the column is an `ENUM` type, the number of `ENUM` values in the target cluster must be a superset of that in the backup data. ## Performance and impact @@ -191,7 +187,7 @@ To illustrate the impact of backup, this document lists the test conclusions of You can use the following methods to manually control the impact of backup tasks on cluster performance. However, these two methods also reduce the speed of backup tasks while reducing the impact of backup tasks on the cluster. -- Use the `--ratelimit` parameter to limit the speed of backup tasks. Note that this parameter limits the speed of **saving backup files to external storage**. When calculating the total size of backup files, use the `backup data size(after compressed)` as a benchmark. +- Use the `--ratelimit` parameter to limit the speed of backup tasks. Note that this parameter limits the speed of **saving backup files to external storage**. When calculating the total size of backup files, use the `backup data size(after compressed)` as a benchmark. When `--ratelimit` is set, to avoid too many tasks causing the speed limit to fail, the `concurrency` parameter of br is automatically adjusted to `1`. - Adjust the TiKV configuration item [`backup.num-threads`](/tikv-configuration-file.md#num-threads-1) to limit the number of threads used by backup tasks. According to internal tests, when BR uses no more than `8` threads for backup tasks, and the total CPU utilization of the cluster does not exceed 60%, the backup tasks have little impact on the cluster, regardless of the read and write workload. The impact of backup on cluster performance can be reduced by limiting the backup threads number, but this affects the backup performance. The preceding tests show that the backup speed is proportional to the number of backup threads. When the number of threads is small, the backup speed is about 20 MiB/thread. For example, 5 backup threads on a single TiKV node can reach a backup speed of 100 MiB/s. diff --git a/br/br-snapshot-manual.md b/br/br-snapshot-manual.md index d7585da5142e7..b3eac3689c887 100644 --- a/br/br-snapshot-manual.md +++ b/br/br-snapshot-manual.md @@ -1,6 +1,6 @@ --- title: TiDB Snapshot Backup and Restore Command Manual -summary: Learn about the commands of TiDB snapshot backup and restore. +summary: TiDB Snapshot Backup and Restore Command Manual describes commands for backing up and restoring cluster snapshots, databases, and tables. It also covers encrypting backup data and restoring encrypted snapshots. The BR tool supports self-adapting to GC and introduces the --ignore-stats parameter for backing up and restoring statistics. It also supports encrypting backup data and restoring partial data of specified databases or tables. --- # TiDB Snapshot Backup and Restore Command Manual @@ -34,7 +34,7 @@ You can back up the latest or specified snapshot of the TiDB cluster using the ` ```shell br backup full \ --pd "${PD_IP}:2379" \ - --backupts '2022-09-08 13:30:00' \ + --backupts '2024-06-28 13:30:00 +08:00' \ --storage "s3://${backup_collection_addr}/snapshot-${date}?access-key=${access-key}&secret-access-key=${secret-access-key}" \ --ratelimit 128 \ --log-file backupfull.log @@ -42,7 +42,7 @@ br backup full \ In the preceding command: -- `--backupts`: The time point of the snapshot. The format can be [TSO](/glossary.md#tso) or timestamp, such as `400036290571534337` or `2018-05-11 01:42:23`. If the data of this snapshot is garbage collected, the `br backup` command returns an error and 'br' exits. If you leave this parameter unspecified, `br` picks the snapshot corresponding to the backup start time. +- `--backupts`: The time point of the snapshot. The format can be [TSO](/glossary.md#tso) or timestamp, such as `400036290571534337` or `2018-05-11 01:42:23 +08:00`. If the data of this snapshot is garbage collected, the `br backup` command returns an error and 'br' exits. If you leave this parameter unspecified, `br` picks the snapshot corresponding to the backup start time. - `--ratelimit`: The maximum speed **per TiKV** performing backup tasks. The unit is in MiB/s. - `--log-file`: The target file where `br` log is written. @@ -116,6 +116,10 @@ Starting from TiDB v7.5.0, the `br` command-line tool introduces the `--ignore-s If you do not set this parameter to `false`, the `br` command-line tool uses the default setting `--ignore-stats=true`, which means statistics are not backed up during data backup. +> **Warning:** +> +> In TiDB v7.5.0, when you use the `br` command-line tool to restore statistics, it requires a large amount of memory, especially when the task involves a large number of tables. Therefore, it is recommended to increase the memory on the node where `br` is running to ensure correct operation. + The following is an example of backing up cluster snapshot data and backing up table statistics with `--ignore-stats=false`: ```shell @@ -178,7 +182,7 @@ br restore full \ In the preceding command: - `--with-sys-table`: BR restores **data in some system tables**, including account permission data and SQL bindings, and statistics (see [Back up statistics](/br/br-snapshot-manual.md#back-up-statistics)). However, it does not restore statistics tables (`mysql.stat_*`) and system variable tables (`mysql.tidb` and `mysql.global_variables`). For more information, see [Restore tables in the `mysql` schema](/br/br-snapshot-guide.md#restore-tables-in-the-mysql-schema). -- `--ratelimit`: The maximum speed **per TiKV** performing backup tasks. The unit is in MiB/s. +- `--ratelimit`: The maximum speed **per TiKV** performing restore tasks. The unit is in MiB/s. - `--log-file`: The target file where the `br` log is written. During restore, a progress bar is displayed in the terminal as shown below. When the progress bar advances to 100%, the restore task is completed. Then `br` will verify the restored data to ensure data security. diff --git a/br/br-use-overview.md b/br/br-use-overview.md index 487549010e6ae..702ba07545a19 100644 --- a/br/br-use-overview.md +++ b/br/br-use-overview.md @@ -1,7 +1,6 @@ --- title: Usage Overview of TiDB Backup and Restore -summary: Learn about how to deploy the backup and restore tool, and how to use it to back up and restore a TiDB cluster. -aliases: ['/tidb/dev/br-deployment/'] +summary: TiDB Backup and Restore provides best practices for choosing backup methods, managing backup data, and deploying the tool. It recommends using both full and log backups, storing data in recommended storage systems, and setting backup retention periods. The tool can be deployed using the command-line tool, SQL statements, or TiDB Operator on Kubernetes. For detailed usage, refer to the provided documentation. --- # Usage Overview of TiDB Backup and Restore diff --git a/br/rawkv-backup-and-restore.md b/br/rawkv-backup-and-restore.md index 54d5564b808af..d002fb5b0dbf8 100644 --- a/br/rawkv-backup-and-restore.md +++ b/br/rawkv-backup-and-restore.md @@ -1,6 +1,6 @@ --- title: Back Up and Restore RawKV -summary: Learn how to back up and restore RawKV using BR. +summary: TiKV and PD can form a KV database known as RawKV without TiDB. TiKV-BR supports data backup and restore for RawKV. For more details, visit the TiKV-BR User Docs on the TiKV website. --- # Back Up and Restore RawKV diff --git a/br/use-br-command-line-tool.md b/br/use-br-command-line-tool.md index c9bd3da4f16d9..40ef11381c5b0 100644 --- a/br/use-br-command-line-tool.md +++ b/br/use-br-command-line-tool.md @@ -1,6 +1,6 @@ --- title: br Command-line Manual -summary: Learn about the description, options, and usage of the br command-line tool. +summary: The `br` command-line tool is used for snapshot backup, log backup, and point-in-time recovery (PITR) in TiDB clusters. It consists of sub-commands, options, and parameters, with common options like `--pd` for PD service address and `-s` for storage path. Sub-commands include `br backup`, `br log`, and `br restore`, each with specific functionalities. Backup commands include `full`, `db`, and `table` options, while log backup and restore commands have various tasks for managing backup operations. --- # br Command-line Manual @@ -48,6 +48,8 @@ A `br` command consists of multiple layers of sub-commands. Currently, br comman * `--key`: specifies the path to the SSL certificate key in the PEM format. * `--status-addr`: specifies the listening address through which `br` provides statistics to Prometheus. * `--concurrency`: the number of concurrent tasks during the backup or restore. +* `--compression`:determines the compression algorithm used for generating backup files. It supports `lz4`, `snappy`, and `zstd`, with the default being `zstd` (usually no need to modify). For guidance on choosing different compression algorithms, refer to [this document](https://github.com/EighteenZi/rocksdb_wiki/blob/master/Compression.md). +* `--compression-level`:sets the compression level corresponding to the chosen compression algorithm for backup. The default compression level for `zstd` is 3. In most cases there is no need to set this option. ## Commands of full backup diff --git a/cached-tables.md b/cached-tables.md index 4691d3a9feeaa..ed3d9d39add6c 100644 --- a/cached-tables.md +++ b/cached-tables.md @@ -13,9 +13,9 @@ This document describes the usage scenarios of cached tables, the examples, and The cached table feature is suitable for tables with the following characteristics: -- The data volume of the table is small. -- The table is read-only or rarely updated. -- The table is frequently accessed, and you expect a better read performance. +- The data volume of the table is small, for example, less than 4 MiB. +- The table is read-only or rarely updated, for example, with a write QPS (queries per second) of less than 10 times per minute. +- The table is frequently accessed, and you expect a better read performance, for example, when encountering hotspots on small tables during direct reads from from TiKV. When the data volume of the table is small but the data is frequently accessed, the data is concentrated on a Region in TiKV and makes it a hotspot Region, which affects the performance. Therefore, the typical usage scenarios of cached tables are as follows: diff --git a/certificate-authentication.md b/certificate-authentication.md index 7f7dd5b160a13..d40f984b74ba8 100644 --- a/certificate-authentication.md +++ b/certificate-authentication.md @@ -1,7 +1,6 @@ --- title: Certificate-Based Authentication for Login summary: Learn the certificate-based authentication used for login. -aliases: ['/docs/dev/certificate-authentication/','/docs/dev/reference/security/cert-based-authentication/'] --- # Certificate-Based Authentication for Login @@ -241,7 +240,7 @@ ssl-ca="path/to/ca-cert.pem" Start TiDB and check logs. If the following information is displayed in the log, the configuration is successful: ``` -[INFO] [server.go:264] ["secure connection is enabled"] ["client verification enabled"=true] +[INFO] [server.go:286] ["mysql protocol server secure connection is enabled"] ["client verification enabled"=true] ``` ### Configure the client to use client certificate @@ -266,9 +265,9 @@ First, connect TiDB using the client to configure the login verification. Then, ### Get user certificate information -The user certificate information can be specified by `require subject`, `require issuer`, `require san`, and `require cipher`, which are used to check the X509 certificate attributes. +The user certificate information can be specified by `REQUIRE SUBJECT`, `REQUIRE ISSUER`, `REQUIRE SAN`, and `REQUIRE CIPHER`, which are used to check the X.509 certificate attributes. -+ `require subject`: Specifies the `subject` information of the client certificate when you log in. With this option specified, you do not need to configure `require ssl` or x509. The information to be specified is consistent with the entered `subject` information in [Generate client keys and certificates](#generate-client-key-and-certificate). ++ `REQUIRE SUBJECT`: Specifies the subject information of the client certificate when you log in. With this option specified, you do not need to configure `require ssl` or x509. The information to be specified is consistent with the entered subject information in [Generate client keys and certificates](#generate-client-key-and-certificate). To get this option, execute the following command: @@ -290,7 +289,7 @@ The user certificate information can be specified by `require subject`, `require + `require san`: Specifies the `Subject Alternative Name` information of the CA certificate that issues the user certificate. The information to be specified is consistent with the [`alt_names` of the `openssl.cnf` configuration file](https://docs.pingcap.com/tidb/stable/generate-self-signed-certificates) used to generate the client certificate. - + Execute the following command to get the information of the `require san` item in the generated certificate: + + Execute the following command to get the information of the `REQUIRE SAN` item in the generated certificate: {{< copyable "shell-regular" >}} @@ -298,25 +297,23 @@ The user certificate information can be specified by `require subject`, `require openssl x509 -noout -extensions subjectAltName -in client.crt ``` - + `require san` currently supports the following `Subject Alternative Name` check items: + + `REQUIRE SAN` currently supports the following `Subject Alternative Name` check items: - URI - IP - DNS - + Multiple check items can be configured after they are connected by commas. For example, configure `require san` as follows for the `u1` user: + + Multiple check items can be configured after they are connected by commas. For example, configure `REQUIRE SAN` as follows for the `u1` user: {{< copyable "sql" >}} ```sql - create user 'u1'@'%' require san 'DNS:d1,URI:spiffe://example.org/myservice1,URI:spiffe://example.org/myservice2'; + CREATE USER 'u1'@'%' REQUIRE SAN 'DNS:d1,URI:spiffe://example.org/myservice1,URI:spiffe://example.org/myservice2'; ``` The above configuration only allows the `u1` user to log in to TiDB using the certificate with the URI item `spiffe://example.org/myservice1` or `spiffe://example.org/myservice2` and the DNS item `d1`. -+ `require cipher`: Checks the cipher method supported by the client. Use the following statement to check the list of supported cipher methods: - - {{< copyable "sql" >}} ++ `REQUIRE CIPHER`: Checks the cipher method supported by the client. Use the following statement to check the list of supported cipher methods: ```sql SHOW SESSION STATUS LIKE 'Ssl_cipher_list'; @@ -324,24 +321,16 @@ The user certificate information can be specified by `require subject`, `require ### Configure user certificate information -After getting the user certificate information (`require subject`, `require issuer`, `require san`, `require cipher`), configure these information to be verified when creating a user, granting privileges, or altering a user. Replace `` with the corresponding information in the following statements. +After getting the user certificate information (`REQUIRE SUBJECT`, `REQUIRE ISSUER`, `REQUIRE SAN`, `REQUIRE CIPHER`), configure these information to be verified when creating a user, granting privileges, or altering a user. Replace `` with the corresponding information in the following statements. You can configure one option or multiple options using the space or `and` as the separator. -+ Configure user certificate when creating a user (`create user`): - - {{< copyable "sql" >}} - - ```sql - create user 'u1'@'%' require issuer '' subject '' san '' cipher ''; - ``` - -+ Configure user certificate when granting privileges: ++ Configure user certificate when creating a user (`CREATE USER`): {{< copyable "sql" >}} ```sql - grant all on *.* to 'u1'@'%' require issuer '' subject '' san '' cipher ''; + CREATE USER 'u1'@'%' REQUIRE ISSUER '' SUBJECT '' SAN '' CIPHER ''; ``` + Configure user certificate when altering a user: @@ -349,22 +338,20 @@ You can configure one option or multiple options using the space or `and` as the {{< copyable "sql" >}} ```sql - alter user 'u1'@'%' require issuer '' subject '' san '' cipher ''; + ALTER USER 'u1'@'%' REQUIRE ISSUER '' SUBJECT '' SAN '' CIPHER ''; ``` After the above configuration, the following items will be verified when you log in: + SSL is used; the CA that issues the client certificate is consistent with the CA configured in the server. -+ The `issuer` information of the client certificate matches the information specified in `require issuer`. -+ The `subject` information of the client certificate matches the information specified in `require cipher`. -+ The `Subject Alternative Name` information of the client certificate matches the information specified in `require san`. ++ The `issuer` information of the client certificate matches the information specified in `REQUIRE ISSUER`. ++ The `subject` information of the client certificate matches the information specified in `REQUIRE CIPHER`. ++ The `Subject Alternative Name` information of the client certificate matches the information specified in `REQUIRE SAN`. You can log into TiDB only after all the above items are verified. Otherwise, the `ERROR 1045 (28000): Access denied` error is returned. You can use the following command to check the TLS version, the cipher algorithm and whether the current connection uses the certificate for the login. Connect the MySQL client and execute the following statement: -{{< copyable "sql" >}} - ```sql \s ``` @@ -373,20 +360,18 @@ The output: ``` -------------- -mysql Ver 15.1 Distrib 10.4.10-MariaDB, for Linux (x86_64) using readline 5.1 +mysql Ver {{{ .tidb-version }}} for Linux on x86_64 (MySQL Community Server - GPL) Connection id: 1 Current database: test Current user: root@127.0.0.1 -SSL: Cipher in use is TLS_AES_256_GCM_SHA384 +SSL: Cipher in use is TLS_AES_128_GCM_SHA256 ``` Then execute the following statement: -{{< copyable "sql" >}} - ```sql -show variables like '%ssl%'; +SHOW VARIABLES LIKE '%ssl%'; ``` The output: @@ -395,13 +380,14 @@ The output: +---------------+----------------------------------+ | Variable_name | Value | +---------------+----------------------------------+ -| ssl_cert | /path/to/server-cert.pem | -| ssl_ca | /path/to/ca-cert.pem | -| have_ssl | YES | | have_openssl | YES | +| have_ssl | YES | +| ssl_ca | /path/to/ca-cert.pem | +| ssl_cert | /path/to/server-cert.pem | +| ssl_cipher | | | ssl_key | /path/to/server-key.pem | +---------------+----------------------------------+ -6 rows in set (0.067 sec) +6 rows in set (0.06 sec) ``` ## Update and replace certificate diff --git a/character-set-and-collation.md b/character-set-and-collation.md index 6ac04f4dd2ff5..6da9616ce997c 100644 --- a/character-set-and-collation.md +++ b/character-set-and-collation.md @@ -1,7 +1,6 @@ --- title: Character Set and Collation summary: Learn about the supported character sets and collations in TiDB. -aliases: ['/docs/dev/character-set-and-collation/','/docs/dev/reference/sql/characterset-and-collation/','/docs/dev/reference/sql/character-set/'] --- # Character Set and Collation @@ -10,7 +9,7 @@ This document introduces the character sets and collations supported by TiDB. ## Concepts -A character set is a set of symbols and encodings. The default character set in TiDB is utf8mb4, which matches the default in MySQL 8.0 and above. +A character set is a set of symbols and encodings. The default character set in TiDB is `utf8mb4`, which matches the default character set in MySQL 8.0 and later. A collation is a set of rules for comparing characters in a character set, and the sorting order of characters. For example in a binary collation `A` and `a` do not compare as equal: @@ -57,8 +56,6 @@ SELECT 'A' = 'a'; 1 row in set (0.00 sec) ``` -TiDB defaults to using a binary collation. This differs from MySQL, which uses a case-insensitive collation by default. - ## Character sets and collations supported by TiDB Currently, TiDB supports the following character sets: @@ -75,7 +72,7 @@ SHOW CHARACTER SET; +---------+-------------------------------------+-------------------+--------+ | ascii | US ASCII | ascii_bin | 1 | | binary | binary | binary | 1 | -| gbk | Chinese Internal Code Specification | gbk_bin | 2 | +| gbk | Chinese Internal Code Specification | gbk_chinese_ci | 2 | | latin1 | Latin1 | latin1_bin | 1 | | utf8 | UTF-8 Unicode | utf8_bin | 3 | | utf8mb4 | UTF-8 Unicode | utf8mb4_bin | 4 | @@ -114,11 +111,16 @@ SHOW COLLATION; > > TiDB incorrectly treats latin1 as a subset of utf8. This can lead to unexpected behaviors when you store characters that differ between latin1 and utf8 encodings. It is strongly recommended to the utf8mb4 character set. See [TiDB #18955](https://github.com/pingcap/tidb/issues/18955) for more details. > -> If the predicates include `LIKE` for string prefixes, such as `LIKE 'prefix%'`, and the target column is set to a non-binary collation (the suffix does not end with `_bin`), the optimizer currently cannot convert this predicate into a range scan. Instead, it performs a full scan. As a result, such SQL queries might lead to unexpected resource consumption. +> In TiDB v7.5.0, if the predicates include `LIKE` for string prefixes, such as `LIKE 'prefix%'`, and the target column is set to a non-binary collation (the suffix does not end with `_bin`), the optimizer currently cannot convert this predicate into a range scan. Instead, it performs a full scan. As a result, such SQL queries might lead to unexpected resource consumption. Starting from v7.5.1, this limitation is removed. > **Note:** > > The default collations in TiDB (binary collations, with the suffix `_bin`) are different than [the default collations in MySQL](https://dev.mysql.com/doc/refman/8.0/en/charset-charsets.html) (typically general collations, with the suffix `_general_ci` or `_ai_ci`). This can cause incompatible behavior when specifying an explicit character set but relying on the implicit default collation to be chosen. +> +> However, the default collations in TiDB are also affected by the [connection collation](https://dev.mysql.com/doc/refman/8.0/en/charset-connection.html#charset-connection-system-variables) settings of your clients. For example, the MySQL 8.x client defaults to `utf8mb4_0900_ai_ci` as the connection collation for the `utf8mb4` character set. +> +> - Before TiDB v7.4.0, if your client uses `utf8mb4_0900_ai_ci` as the [connection collation](https://dev.mysql.com/doc/refman/8.0/en/charset-connection.html#charset-connection-system-variables), TiDB falls back to using the TiDB server default collation `utf8mb4_bin` because TiDB does not support the `utf8mb4_0900_ai_ci` collation. +> - Starting from v7.4.0, if your client uses `utf8mb4_0900_ai_ci` as the [connection collation](https://dev.mysql.com/doc/refman/8.0/en/charset-connection.html#charset-connection-system-variables), TiDB follows the client's configuration to use `utf8mb4_0900_ai_ci` as the default collation. You can use the following statement to view the collations (under the [new framework for collations](#new-framework-for-collations)) that corresponds to the character set. @@ -412,7 +414,7 @@ You can use the following statement to set the character set and collation that ```sql SET character_set_client = charset_name; SET character_set_results = charset_name; - SET charset_connection = @@charset_database; + SET character_set_connection=@@character_set_database; SET collation_connection = @@collation_database; ``` @@ -496,9 +498,13 @@ Since TiDB v4.0, a complete framework for collations is introduced. -This new framework supports semantically parsing collations and introduces the `new_collations_enabled_on_first_bootstrap` configuration item to decide whether to enable the new framework when a cluster is first initialized. To enable the new framework, set `new_collations_enabled_on_first_bootstrap` to `true`. For details, see [`new_collations_enabled_on_first_bootstrap`](/tidb-configuration-file.md#new_collations_enabled_on_first_bootstrap). If you initialize the cluster after the configuration item is enabled, you can check whether the new collation is enabled through the `new_collation_enabled` variable in the `mysql`.`tidb` table: +This new framework supports semantically parsing collations and introduces the `new_collations_enabled_on_first_bootstrap` configuration item to decide whether to enable the new framework when a cluster is first initialized. To enable the new framework, set `new_collations_enabled_on_first_bootstrap` to `true`. For details, see [`new_collations_enabled_on_first_bootstrap`](/tidb-configuration-file.md#new_collations_enabled_on_first_bootstrap). -{{< copyable "sql" >}} +For a TiDB cluster that is already initialized, you can check whether the new collation is enabled through the `new_collation_enabled` variable in the `mysql.tidb` table: + +> **Note:** +> +> If the query result of the `mysql.tidb` table is different from the value of `new_collations_enabled_on_first_bootstrap`, the result of the `mysql.tidb` table is the actual value. ```sql SELECT VARIABLE_VALUE FROM mysql.tidb WHERE VARIABLE_NAME='new_collation_enabled'; @@ -548,7 +554,7 @@ INSERT INTO t VALUES ('a'); ``` ```sql -ERROR 1062 (23000): Duplicate entry 'a' for key 't.PRIMARY' # TiDB is compatible with the case-insensitive collation of MySQL. +ERROR 1062 (23000): Duplicate entry 'a' for key 't.PRIMARY' -- TiDB is compatible with the case-insensitive collation of MySQL. ``` ```sql @@ -556,7 +562,7 @@ INSERT INTO t VALUES ('a '); ``` ```sql -ERROR 1062 (23000): Duplicate entry 'a ' for key 't.PRIMARY' # TiDB modifies the `PADDING` behavior to be compatible with MySQL. +ERROR 1062 (23000): Duplicate entry 'a ' for key 't.PRIMARY' -- TiDB modifies the `PADDING` behavior to be compatible with MySQL. ``` > **Note:** diff --git a/character-set-gbk.md b/character-set-gbk.md index 75170e3194ea1..5b3f73baff95c 100644 --- a/character-set-gbk.md +++ b/character-set-gbk.md @@ -5,14 +5,16 @@ summary: This document provides details about the TiDB support of the GBK charac # GBK -Since v5.4.0, TiDB supports the GBK character set. This document provides the TiDB support and compatibility information of the GBK character set. +Starting from v5.4.0, TiDB supports the GBK character set. This document provides the TiDB support and compatibility information of the GBK character set. + +Starting from v6.0.0, TiDB enables the [new framework for collations](/character-set-and-collation.md#new-framework-for-collations) by default. The default collation for TiDB GBK character set is `gbk_chinese_ci`, which is consistent with MySQL. ```sql SHOW CHARACTER SET WHERE CHARSET = 'gbk'; +---------+-------------------------------------+-------------------+--------+ | Charset | Description | Default collation | Maxlen | +---------+-------------------------------------+-------------------+--------+ -| gbk | Chinese Internal Code Specification | gbk_bin | 2 | +| gbk | Chinese Internal Code Specification | gbk_chinese_ci | 2 | +---------+-------------------------------------+-------------------+--------+ 1 row in set (0.00 sec) @@ -21,8 +23,9 @@ SHOW COLLATION WHERE CHARSET = 'gbk'; | Collation | Charset | Id | Default | Compiled | Sortlen | +----------------+---------+------+---------+----------+---------+ | gbk_bin | gbk | 87 | | Yes | 1 | +| gbk_chinese_ci | gbk | 28 | Yes | Yes | 1 | +----------------+---------+------+---------+----------+---------+ -1 rows in set (0.00 sec) +2 rows in set (0.00 sec) ``` ## MySQL compatibility @@ -31,40 +34,22 @@ This section provides the compatibility information between MySQL and TiDB. ### Collations -The default collation of the GBK character set in MySQL is `gbk_chinese_ci`. Unlike MySQL, the default collation of the GBK character set in TiDB is `gbk_bin`. Additionally, because TiDB converts GBK to UTF8MB4 and then uses a binary collation, the `gbk_bin` collation in TiDB is not the same as the `gbk_bin` collation in MySQL. - -To make TiDB compatible with the collations of MySQL GBK character set, when you first initialize the TiDB cluster, you need to set the TiDB option [`new_collations_enabled_on_first_bootstrap`](/tidb-configuration-file.md#new_collations_enabled_on_first_bootstrap) to `true` to enable the [new framework for collations](/character-set-and-collation.md#new-framework-for-collations). +The default collation of the GBK character set in MySQL is `gbk_chinese_ci`. The default collation for the GBK character set in TiDB depends on the value of the TiDB configuration item [`new_collations_enabled_on_first_bootstrap`](/tidb-configuration-file.md#new_collations_enabled_on_first_bootstrap): + +- By default, the TiDB configuration item [`new_collations_enabled_on_first_bootstrap`](/tidb-configuration-file.md#new_collations_enabled_on_first_bootstrap) is set to `true`, which means that the [new framework for collations](/character-set-and-collation.md#new-framework-for-collations) is enabled and the default collation for the GBK character set is `gbk_chinese_ci`. +- When the TiDB configuration item [`new_collations_enabled_on_first_bootstrap`](/tidb-configuration-file.md#new_collations_enabled_on_first_bootstrap) is set to `false`, the [new framework for collations](/character-set-and-collation.md#new-framework-for-collations) is disabled, and the default collation for the GBK character set is `gbk_bin`. -To make TiDB compatible with the collations of MySQL GBK character set, when you first initialize the TiDB cluster, TiDB Cloud enables the [new framework for collations](/character-set-and-collation.md#new-framework-for-collations) by default. +By default, TiDB Cloud enables the [new framework for collations](/character-set-and-collation.md#new-framework-for-collations) and the default collation for the GBK character set is `gbk_chinese_ci`. -After enabling the new framework for collations, if you check the collations corresponding to the GBK character set, you can see that the TiDB GBK default collation is changed to `gbk_chinese_ci`. - -```sql -SHOW CHARACTER SET WHERE CHARSET = 'gbk'; -+---------+-------------------------------------+-------------------+--------+ -| Charset | Description | Default collation | Maxlen | -+---------+-------------------------------------+-------------------+--------+ -| gbk | Chinese Internal Code Specification | gbk_chinese_ci | 2 | -+---------+-------------------------------------+-------------------+--------+ -1 row in set (0.00 sec) - -SHOW COLLATION WHERE CHARSET = 'gbk'; -+----------------+---------+------+---------+----------+---------+ -| Collation | Charset | Id | Default | Compiled | Sortlen | -+----------------+---------+------+---------+----------+---------+ -| gbk_bin | gbk | 87 | | Yes | 1 | -| gbk_chinese_ci | gbk | 28 | Yes | Yes | 1 | -+----------------+---------+------+---------+----------+---------+ -2 rows in set (0.00 sec) -``` +Additionally, because TiDB converts GBK to `utf8mb4` and then uses a binary collation, the `gbk_bin` collation in TiDB is not the same as the `gbk_bin` collation in MySQL. ### Illegal character compatibility @@ -98,7 +83,7 @@ In the above table, the result of `SELECT HEX('a');` in the `utf8mb4` byte set i - Currently, for binary characters of the `ENUM` and `SET` types, TiDB deals with them as the `utf8mb4` character set. -- If the predicates include `LIKE` for string prefixes, such as `LIKE 'prefix%'`, and the target column is set to a GBK collation (either `gbk_bin` or `gbk_chinese_ci`), the optimizer currently cannot convert this predicate into a range scan. Instead, it performs a full scan. As a result, such SQL queries might lead to unexpected resource consumption. +- In TiDB v7.5.0, if the predicates include `LIKE` for string prefixes, such as `LIKE 'prefix%'`, and the target column is set to a GBK collation (either `gbk_bin` or `gbk_chinese_ci`), the optimizer currently cannot convert this predicate into a range scan. Instead, it performs a full scan. As a result, such SQL queries might lead to unexpected resource consumption. Starting from v7.5.1, this limitation is removed. ## Component compatibility diff --git a/check-before-deployment.md b/check-before-deployment.md index 38a5e3b52b935..1e158a9881fd7 100644 --- a/check-before-deployment.md +++ b/check-before-deployment.md @@ -1,7 +1,6 @@ --- title: TiDB Environment and System Configuration Check summary: Learn the environment check operations before deploying TiDB. -aliases: ['/docs/dev/check-before-deployment/'] --- # TiDB Environment and System Configuration Check @@ -42,6 +41,13 @@ Take the `/dev/nvme0n1` data disk as an example: parted -s -a optimal /dev/nvme0n1 mklabel gpt -- mkpart primary ext4 1 -1 ``` + For large NVMe devices, you can create multiple partitions: + + ```bash + parted -s -a optimal /dev/nvme0n1 mklabel gpt -- mkpart primary ext4 1 2000GB + parted -s -a optimal /dev/nvme0n1 -- mkpart primary ext4 2000GB -1 + ``` + > **Note:** > > Use the `lsblk` command to view the device number of the partition: for a NVMe disk, the generated device number is usually `nvme0n1p1`; for a regular disk (for example, `/dev/sdb`), the generated device number is usually `sdb1`. @@ -93,6 +99,7 @@ Take the `/dev/nvme0n1` data disk as an example: ```bash mkdir /data1 && \ + systemctl daemon-reload && \ mount -a ``` @@ -112,21 +119,25 @@ Take the `/dev/nvme0n1` data disk as an example: ## Check and disable system swap -TiDB needs sufficient memory space for operation. When memory is insufficient, using swap as a buffer might degrade performance. Therefore, it is recommended to disable the system swap permanently by executing the following commands: +TiDB needs a sufficient amount of memory for operation. If the memory that TiDB uses gets swapped out and later gets swapped back in, this can cause latency spikes. If you want to maintain stable performance, it is recommended that you permanently disable the system swap, but it might trigger OOM issues when there is insufficient memory. If you want to avoid such OOM issues, you can just decrease the swap priority, instead of permanently disabling it. -{{< copyable "shell-regular" >}} +- Enabling and using swap might introduce performance jitter issues. It is recommended that you permanently disable the operating system tier swap for low-latency and stability-critical database services. To permanently disable swap, you can use the following method: -```bash -echo "vm.swappiness = 0">> /etc/sysctl.conf -swapoff -a && swapon -a -sysctl -p -``` + - During the initialization phase of the operating system, do not partition the swap partition disk separately. + - If you have already partitioned a separate swap partition disk during the initialization phase of the operating system and enabled swap, run the following command to disable it: -> **Note:** -> -> - Executing `swapoff -a` and then `swapon -a` is to refresh swap by dumping data to memory and cleaning up swap. If you drop the swappiness change and execute only `swapoff -a`, swap will be enabled again after you restart the system. -> -> - `sysctl -p` is to make the configuration effective without restarting the system. + ```bash + echo "vm.swappiness = 0">> /etc/sysctl.conf + sysctl -p + swapoff -a && swapon -a + ``` + +- If the host memory is insufficient, disabling the system swap might be more likely to trigger OOM issues. You can run the following command to decrease the swap priority instead of disabling it permanently: + + ```bash + echo "vm.swappiness = 0">> /etc/sysctl.conf + sysctl -p + ``` ## Set temporary spaces for TiDB instances (Recommended) @@ -138,25 +149,25 @@ Some operations in TiDB require writing temporary files to the server, so it is - `Fast Online DDL` work area - When the variable [`tidb_ddl_enable_fast_reorg`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) is set to `ON` (the default value in v6.5.0 and later versions), `Fast Online DDL` is enabled, and some DDL operations need to read and write temporary files in filesystems. The location is defined by the configuration item [`temp-dir`](/tidb-configuration-file.md#temp-dir-new-in-v630). You need to ensure that the user that runs TiDB has read and write permissions for that directory of the operating system. Taking the default directory `/tmp/tidb` as an example: + When the variable [`tidb_ddl_enable_fast_reorg`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) is set to `ON` (the default value in v6.5.0 and later versions), `Fast Online DDL` is enabled, and some DDL operations need to read and write temporary files in filesystems. The location is defined by the configuration item [`temp-dir`](/tidb-configuration-file.md#temp-dir-new-in-v630). You need to ensure that the user that runs TiDB has read and write permissions for that directory of the operating system. The default directory `/tmp/tidb` uses tmpfs (temporary file system). It is recommended to explicitly specify a disk directory. The following uses `/data/tidb-deploy/tempdir` as an example: > **Note:** > > If DDL operations on large objects exist in your application, it is highly recommended to configure an independent large file system for [`temp-dir`](/tidb-configuration-file.md#temp-dir-new-in-v630). ```shell - sudo mkdir /tmp/tidb + sudo mkdir -p /data/tidb-deploy/tempdir ``` - If the `/tmp/tidb` directory already exists, make sure the write permission is granted. + If the `/data/tidb-deploy/tempdir` directory already exists, make sure the write permission is granted. ```shell - sudo chmod -R 777 /tmp/tidb + sudo chmod -R 777 /data/tidb-deploy/tempdir ``` > **Note:** > - > If the directory does not exist, TiDB will automatically create it upon startup. If the directory creation fails or TiDB does not have the read and write permissions for that directory, [`Fast Online DDL`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) might experience unpredictable issues during runtime. + > If the directory does not exist, TiDB will automatically create it upon startup. If the directory creation fails or TiDB does not have the read and write permissions for that directory, [`Fast Online DDL`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) will be disabled during runtime. ## Check and stop the firewall service of target machines @@ -269,7 +280,7 @@ To check whether the NTP service is installed and whether it synchronizes with t Unable to talk to NTP daemon. Is it running? ``` -3. Run the `chronyc tracking` command to check wheter the Chrony service synchronizes with the NTP server. +3. Run the `chronyc tracking` command to check whether the Chrony service synchronizes with the NTP server. > **Note:** > @@ -336,7 +347,11 @@ sudo systemctl enable ntpd.service For TiDB in the production environment, it is recommended to optimize the operating system configuration in the following ways: 1. Disable THP (Transparent Huge Pages). The memory access pattern of databases tends to be sparse rather than consecutive. If the high-level memory fragmentation is serious, higher latency will occur when THP pages are allocated. -2. Set the I/O Scheduler of the storage media to `noop`. For the high-speed SSD storage media, the kernel's I/O scheduling operations can cause performance loss. After the Scheduler is set to `noop`, the performance is better because the kernel directly sends I/O requests to the hardware without other operations. Also, the noop Scheduler is better applicable. +2. Set the I/O Scheduler of the storage media. + + - For the high-speed SSD storage, the kernel's default I/O scheduling operations might cause performance loss. It is recommended to set the I/O Scheduler to first-in-first-out (FIFO), such as `noop` or `none`. This configuration allows the kernel to pass I/O requests directly to hardware without scheduling, thus improving performance. + - For NVMe storage, the default I/O Scheduler is `none`, so no adjustment is needed. + 3. Choose the `performance` mode for the cpufrequ module which controls the CPU frequency. The performance is maximized when the CPU frequency is fixed at its highest supported operating frequency without dynamic adjustment. Take the following steps to check the current operating system configuration and configure optimal parameters: @@ -357,9 +372,9 @@ Take the following steps to check the current operating system configuration and > > If `[always] madvise never` is output, THP is enabled. You need to disable it. -2. Execute the following command to see the I/O Scheduler of the disk where the data directory is located. Assume that you create data directories on both sdb and sdc disks: +2. Execute the following command to see the I/O Scheduler of the disk where the data directory is located. - {{< copyable "shell-regular" >}} + If your data directory uses an SD or VD device, run the following command to check the I/O Scheduler: ```bash cat /sys/block/sd[bc]/queue/scheduler @@ -374,6 +389,21 @@ Take the following steps to check the current operating system configuration and > > If `noop [deadline] cfq` is output, the I/O Scheduler for the disk is in the `deadline` mode. You need to change it to `noop`. + If your data directory uses an NVMe device, run the following command to check the I/O Scheduler: + + ```bash + cat /sys/block/nvme[01]*/queue/scheduler + ``` + + ``` + [none] mq-deadline kyber bfq + [none] mq-deadline kyber bfq + ``` + + > **Note:** + > + > `[none] mq-deadline kyber bfq` indicates that the NVMe device uses the `none` I/O Scheduler, and no changes are needed. + 3. Execute the following command to see the `ID_SERIAL` of the disk: {{< copyable "shell-regular" >}} @@ -389,7 +419,8 @@ Take the following steps to check the current operating system configuration and > **Note:** > - > If multiple disks are allocated with data directories, you need to execute the above command several times to record the `ID_SERIAL` of each disk. + > - If multiple disks are allocated with data directories, you need to execute the above command for each disk to record the `ID_SERIAL` of each disk. + > - If your device uses the `noop` or `none` Scheduler, you do not need to record the `ID_SERIAL` or configure udev rules or the tuned profile. 4. Execute the following command to see the power policy of the cpufreq module: @@ -466,6 +497,10 @@ Take the following steps to check the current operating system configuration and 3. Apply the new tuned profile: + > **Note:** + > + > If your device uses the `noop` or `none` I/O Scheduler, skip this step. No Scheduler configuration is needed in the tuned profile. + {{< copyable "shell-regular" >}} ```bash @@ -495,12 +530,12 @@ Take the following steps to check the current operating system configuration and {{< copyable "shell-regular" >}} ```bash - grubby --args="transparent_hugepage=never" --update-kernel /boot/vmlinuz-3.10.0-957.el7.x86_64 + grubby --args="transparent_hugepage=never" --update-kernel `grubby --default-kernel` ``` > **Note:** > - > `--update-kernel` is followed by the actual default kernel version. + > You can also specify the actual version number after `--update-kernel`, for example, `--update-kernel /boot/vmlinuz-3.10.0-957.el7.x86_64`. 3. Execute `grubby --info` to see the modified default kernel configuration: @@ -548,6 +583,10 @@ Take the following steps to check the current operating system configuration and 6. Apply the udev script: + > **Note:** + > + > If your device uses the `noop` or `none` I/O Scheduler, skip this step. No udev rules configuration is needed. + {{< copyable "shell-regular" >}} ```bash @@ -630,9 +669,18 @@ Take the following steps to check the current operating system configuration and echo "net.ipv4.tcp_tw_recycle = 0">> /etc/sysctl.conf echo "net.ipv4.tcp_syncookies = 0">> /etc/sysctl.conf echo "vm.overcommit_memory = 1">> /etc/sysctl.conf + echo "vm.min_free_kbytes = 1048576">> /etc/sysctl.conf sysctl -p ``` + > **Note:** + > + > - `vm.min_free_kbytes` is a Linux kernel parameter that controls the minimum amount of free memory reserved by the system, measured in KiB. + > - The setting of `vm.min_free_kbytes` affects the memory reclaim mechanism. Setting it too large reduces the available memory, while setting it too small might cause memory request speeds to exceed background reclaim speeds, leading to memory reclamation and consequent delays in memory allocation. + > - It is recommended to set `vm.min_free_kbytes` to `1048576` KiB (1 GiB) at least. If [NUMA is installed](/check-before-deployment.md#install-the-numactl-tool), it is recommended to set it to `number of NUMA nodes * 1048576` KiB. + > - For servers with memory sizes less than 16 GiB, it is recommended to keep the default value of `vm.min_free_kbytes` unchanged. + > - `tcp_tw_recycle` is removed in Linux kernel 4.12. Skip this setting if you are using a later kernel version. + 10. Execute the following command to configure the user's `limits.conf` file: {{< copyable "shell-regular" >}} diff --git a/choose-index.md b/choose-index.md index 99415cd957754..e30afa919fd73 100644 --- a/choose-index.md +++ b/choose-index.md @@ -40,7 +40,7 @@ TiDB uses the following heuristic pre-rules to select indexes: + Rule 1: If an index satisfies "unique index with full match + no need to retrieve rows from a table (which means that the plan generated by the index is the IndexReader operator)", TiDB directly selects this index. -+ Rule 2: If an index satisfies "unique index with full match + the need to retrieve rows from a table (which means that the plan generated by the index is the IndexReader operator)", TiDB selects the index with the smallest number of rows to be retrieved from a table as a candidate index. ++ Rule 2: If an index satisfies "unique index with full match + the need to retrieve rows from a table (which means that the plan generated by the index is the IndexLookupReader operator)", TiDB selects the index with the smallest number of rows to be retrieved from a table as a candidate index. + Rule 3: If an index satisfies "ordinary index + no need to retrieve rows from a table + the number of rows to be read is less than the value of a certain threshold", TiDB selects the index with the smallest number of rows to be read as a candidate index. diff --git a/clinic/clinic-data-instruction-for-tiup.md b/clinic/clinic-data-instruction-for-tiup.md index b5d8e8535b170..da9bba0883313 100644 --- a/clinic/clinic-data-instruction-for-tiup.md +++ b/clinic/clinic-data-instruction-for-tiup.md @@ -1,6 +1,6 @@ --- title: PingCAP Clinic Diagnostic Data -summary: Learn what diagnostic data can be collected by PingCAP Clinic Diagnostic Service from the TiDB and DM clusters deployed using TiUP. +summary: PingCAP Clinic Diagnostic Service collects diagnostic data from TiDB and DM clusters using TiUP. Data types include cluster information, diagnostic data for TiDB, TiKV, PD, TiFlash, TiCDC, Prometheus monitoring, system variables, and node system information. Data is stored in Clinic Server for international and Chinese mainland users. The collected data is only used for troubleshooting cluster problems. --- # PingCAP Clinic Diagnostic Data @@ -32,6 +32,7 @@ This section lists the types of diagnostic data that can be collected by [Diag]( | Log | `tidb.log` | `--include=log` | | Error log | `tidb_stderr.log` | `--include=log` | | Slow log | `tidb_slow_query.log` | `--include=log` | +| Audit log | `tidb-audit.log.json` | `--include=log` | | Configuration file | `tidb.toml` | `--include=config` | | Real-time configuration | `config.json` | `--include=config` | @@ -112,7 +113,7 @@ This section lists the types of diagnostic data that can be collected by Diag fr | Data type | Exported file | Parameter for data collection by PingCAP Clinic | | :------ | :------ |:-------- | -| Log | `m-master.log` | `--include=log` | +| Log | `dm-master.log` | `--include=log` | | Error log | `dm-master_stderr.log` | `--include=log` | | Configuration file | `dm-master.toml` | `--include=config` | @@ -140,3 +141,14 @@ This section lists the types of diagnostic data that can be collected by Diag fr | Contents in the `/etc/security/limits.conf` system | `limits.conf` | `--include=system` | | List of kernel parameters | `sysctl.conf` | `--include=system` | | Socket system information, which is the output of the `ss` command | `ss.txt` | `--include=system` | + +### Log file classification + +You can use the `--include=log.` parameter to specify which types of logs to collect. + +Log types: + +- `std`: Log files that contain `stderr` in the filename. +- `rocksdb`: Log files with a `rocksdb` prefix and a `.info` suffix. +- `slow`: Slow query log files. +- `unknown`: Log files that do not match any of the preceding types. diff --git a/clinic/clinic-introduction.md b/clinic/clinic-introduction.md index 44cc8db40e988..48b3b00f26b4f 100644 --- a/clinic/clinic-introduction.md +++ b/clinic/clinic-introduction.md @@ -1,6 +1,6 @@ --- title: PingCAP Clinic Overview -summary: Learn about the PingCAP Clinic Diagnostic Service (PingCAP Clinic), including tool components, user scenarios, and implementation principles. +summary: PingCAP Clinic is a diagnostic service for TiDB clusters deployed using TiUP or TiDB Operator. It helps troubleshoot cluster problems remotely, ensures stable operation, and provides quick cluster status checks. The service includes Diag client for data collection and Clinic Server for online diagnostic reports. Users can troubleshoot problems remotely and quickly check cluster status. Diag collects diagnostic data through various methods, and Clinic Server has limitations on clusters, storage, and data size. The service is free until April 15, 2025. Next steps include using PingCAP Clinic in different environments. --- # PingCAP Clinic Overview @@ -61,7 +61,7 @@ First, Diag gets cluster topology information from the deployment tool TiUP (tiu > **Note:** > -> - Clinic Server is free from July 15, 2022 to July 14, 2024. You will be notified through email before July 14, 2024 if the service starts charging fee afterwards. +> - Clinic Server is free from July 15, 2022 to April 15, 2025. You will be notified through email before April 15, 2025 if the service starts charging fee afterwards. > - If you want to adjust the usage limitations, [get support](/support.md) from PingCAP. | Service Type| Limitation | diff --git a/clinic/clinic-user-guide-for-tiup.md b/clinic/clinic-user-guide-for-tiup.md index 7ac8b6f56205e..7496f350fba48 100644 --- a/clinic/clinic-user-guide-for-tiup.md +++ b/clinic/clinic-user-guide-for-tiup.md @@ -1,6 +1,6 @@ --- title: Troubleshoot Clusters Using PingCAP Clinic -summary: Learn how to use the PingCAP Clinic Diagnostic Service to troubleshoot cluster problems remotely and perform a quick check of the cluster status on a TiDB cluster or DM cluster deployed using TiUP. +summary: PingCAP Clinic Diagnostic Service (PingCAP Clinic) helps troubleshoot TiDB and DM clusters deployed using TiUP. It allows remote troubleshooting and local cluster status checks using Diag client and Clinic Server. Prerequisites include installing Diag, setting an access token, and configuring the region. Troubleshooting remotely involves collecting, viewing, and uploading diagnostic data. Performing a quick check on the cluster status locally involves collecting and diagnosing configuration data. Data upload supports breakpoint upload, and uploaded data is kept on the Clinic Server for a maximum of 180 days. --- # Troubleshoot Clusters Using PingCAP Clinic @@ -30,7 +30,7 @@ Before using PingCAP Clinic, you need to install Diag (a component to collect da 1. Install Diag. - - If you have installed TiUP on your control machine, run the following command to install Diag: + - If you have installed TiUP on your control machine, run the following command to install Diag: ```bash tiup install diag @@ -58,7 +58,7 @@ Before using PingCAP Clinic, you need to install Diag (a component to collect da
- [Clinic Server for international users](https://clinic.pingcap.com): Data is stored in AWS in US. + [Clinic Server for international users](https://clinic.pingcap.com): Data is stored in AWS in US regions.
diff --git a/clinic/quick-start-with-clinic.md b/clinic/quick-start-with-clinic.md index 1411036a8d1c6..56538e1aeda43 100644 --- a/clinic/quick-start-with-clinic.md +++ b/clinic/quick-start-with-clinic.md @@ -1,6 +1,6 @@ --- title: Quick Start Guide for PingCAP Clinic -summary: Learn how to use PingCAP Clinic to collect, upload, and view cluster diagnosis data quickly. +summary: PingCAP Clinic is a service for collecting and viewing cluster diagnosis data quickly. It consists of Diag client and Clinic Server. Users can collect diagnostic data with Diag, upload it to Clinic Server, and view the results of Health Report. Before using it, users need to install Diag, log in to Clinic Server, create an organization, get an access token, and set the token and region in Diag. After collecting and uploading data, users can get the data access link and view the Health Report. --- # Quick Start Guide for PingCAP Clinic @@ -34,7 +34,7 @@ Before using PingCAP Clinic, you need to install Diag and prepare an environment
- Go to the [Clinic Server for international users](https://clinic.pingcap.com) and select **Sign in with TiDB Account** to enter the TiDB Cloud login page. If you do not have a TiDB Cloud account, create one on that page. + Go to the [Clinic Server for international users](https://clinic.pingcap.com) and select **Continue with TiDB Account** to enter the TiDB Cloud login page. If you do not have a TiDB Cloud account, create one on that page. > **Note:** > @@ -44,7 +44,7 @@ Before using PingCAP Clinic, you need to install Diag and prepare an environment
- Go to the [Clinic Server for users in the Chinese mainland](https://clinic.pingcap.com.cn) and select **Sign in with AskTUG** to enter the AskTUG community login page. If you do not have an AskTUG account, create one on that page + Go to the [Clinic Server for users in the Chinese mainland](https://clinic.pingcap.com.cn) and select **Continue with AskTUG** to enter the AskTUG community login page. If you do not have an AskTUG account, create one on that page
diff --git a/clustered-indexes.md b/clustered-indexes.md index 31a74827c58fa..0ba8ba6e8d99b 100644 --- a/clustered-indexes.md +++ b/clustered-indexes.md @@ -7,11 +7,11 @@ summary: Learn the concept, user scenarios, usages, limitations, and compatibili TiDB supports the clustered index feature since v5.0. This feature controls how data is stored in tables containing primary keys. It provides TiDB the ability to organize tables in a way that can improve the performance of certain queries. -The term _clustered_ in this context refers to the _organization of how data is stored_ and not _a group of database servers working together_. Some database management systems refer to clustered indexes as _index-organized tables_ (IOT). +The term _clustered_ in this context refers to the _organization of how data is stored_ and not _a group of database servers working together_. Some database management systems refer to clustered index tables as _index-organized tables_ (IOT). Currently, tables containing primary keys in TiDB are divided into the following two categories: -- `NONCLUSTERED`: The primary key of the table is non-clustered index. In tables with non-clustered indexes, the keys for row data consist of internal `_tidb_rowid` implicitly assigned by TiDB. Because primary keys are essentially unique indexes, tables with non-clustered indexes need at least two key-value pairs to store a row, which are: +- `NONCLUSTERED`: The primary key of the table is non-clustered index. In tables with non-clustered indexes, the keys for row data consist of internal [`_tidb_rowid`](https://docs.pingcap.com/tidb/v7.5/tidb-rowid/) values implicitly assigned by TiDB. Because primary keys are essentially unique indexes, tables with non-clustered indexes need at least two key-value pairs to store a row, which are: - `_tidb_rowid` (key) - row data (value) - Primary key data (key) - `_tidb_rowid` (value) - `CLUSTERED`: The primary key of the table is clustered index. In tables with clustered indexes, the keys for row data consist of primary key data given by the user. Therefore, tables with clustered indexes need only one key-value pair to store a row, which is: @@ -37,7 +37,7 @@ On the other hand, tables with clustered indexes have certain disadvantages. See ## Usages -## Create a table with clustered indexes +### Create a table with clustered indexes Since TiDB v5.0, you can add non-reserved keywords `CLUSTERED` or `NONCLUSTERED` after `PRIMARY KEY` in a `CREATE TABLE` statement to specify whether the table's primary key is a clustered index. For example: @@ -143,7 +143,7 @@ mysql> SELECT TIDB_PK_TYPE FROM information_schema.tables WHERE table_schema = ' Currently, there are several different types of limitations for the clustered index feature. See the following: - Situations that are not supported and not in the support plan: - - Using clustered indexes together with the attribute [`SHARD_ROW_ID_BITS`](/shard-row-id-bits.md) is not supported. Also, the attribute [`PRE_SPLIT_REGIONS`](/sql-statements/sql-statement-split-region.md#pre_split_regions) does not take effect on tables with clustered indexes. + - Using clustered indexes together with the attribute [`SHARD_ROW_ID_BITS`](/shard-row-id-bits.md) is not supported. Also, the attribute [`PRE_SPLIT_REGIONS`](/sql-statements/sql-statement-split-region.md#pre_split_regions) does not take effect on tables with clustered indexes that are not [`AUTO_RANDOM`](/auto-random.md). - Downgrading tables with clustered indexes is not supported. If you need to downgrade such tables, use logical backup tools to migrate data instead. - Situations that are not supported yet but in the support plan: - Adding, dropping, and altering clustered indexes using `ALTER TABLE` statements are not supported. diff --git a/command-line-flags-for-pd-configuration.md b/command-line-flags-for-pd-configuration.md index c12a03b8a3bf2..d3a8cb62d460b 100644 --- a/command-line-flags-for-pd-configuration.md +++ b/command-line-flags-for-pd-configuration.md @@ -1,7 +1,6 @@ --- title: PD Configuration Flags summary: Learn some configuration flags of PD. -aliases: ['/docs/dev/command-line-flags-for-pd-configuration/','/docs/dev/reference/configuration/pd-server/configuration/'] --- # PD Configuration Flags @@ -83,7 +82,7 @@ PD is configurable using command-line flags and environment variables. ## `--name` - The human-readable unique name for this PD member -- Default: `"pd"` +- Default: `"pd-${hostname}"` - If you want to start multiply PDs, you must use different name for each one. ## `--cacert` diff --git a/command-line-flags-for-tidb-configuration.md b/command-line-flags-for-tidb-configuration.md index e4b5b936f2e7e..56d02c9562349 100644 --- a/command-line-flags-for-tidb-configuration.md +++ b/command-line-flags-for-tidb-configuration.md @@ -1,7 +1,6 @@ --- title: Configuration Options summary: Learn the configuration options in TiDB. -aliases: ['/docs/dev/command-line-flags-for-tidb-configuration/','/docs/dev/reference/configuration/tidb-server/configuration/','/docs/dev/reference/configuration/tidb-server/server-command-option/'] --- # Configuration Options diff --git a/command-line-flags-for-tikv-configuration.md b/command-line-flags-for-tikv-configuration.md index 86c5e8e810e70..18f2fea7b4a47 100644 --- a/command-line-flags-for-tikv-configuration.md +++ b/command-line-flags-for-tikv-configuration.md @@ -1,7 +1,6 @@ --- title: TiKV Configuration Flags summary: Learn some configuration flags of TiKV. -aliases: ['/docs/dev/command-line-flags-for-tikv-configuration/','/docs/dev/reference/configuration/tikv-server/configuration/'] --- # TiKV Configuration Flags diff --git a/comment-syntax.md b/comment-syntax.md index ac9530cb8091b..0a47026f5b80d 100644 --- a/comment-syntax.md +++ b/comment-syntax.md @@ -1,7 +1,6 @@ --- title: Comment Syntax summary: This document introduces the comment syntax supported by TiDB. -aliases: ['/docs/dev/comment-syntax/','/docs/dev/reference/sql/language-structure/comment-syntax/'] --- # Comment Syntax diff --git a/config-templates/geo-redundancy-deployment.yaml b/config-templates/geo-redundancy-deployment.yaml index 74ad7ecddca3a..4f839c7752cc8 100644 --- a/config-templates/geo-redundancy-deployment.yaml +++ b/config-templates/geo-redundancy-deployment.yaml @@ -107,8 +107,8 @@ tikv_servers: host: host1 readpool.storage.use-unified-pool: true readpool.storage.low-concurrency: 10 - raftstore.raft-min-election-timeout-ticks: 1000 - raftstore.raft-max-election-timeout-ticks: 1020 + raftstore.raft-min-election-timeout-ticks: 50 + raftstore.raft-max-election-timeout-ticks: 60 monitoring_servers: - host: 10.0.1.16 grafana_servers: diff --git a/configure-load-base-split.md b/configure-load-base-split.md index 63abac9ff2194..7958c14d8583e 100644 --- a/configure-load-base-split.md +++ b/configure-load-base-split.md @@ -1,7 +1,6 @@ --- title: Load Base Split summary: Learn the feature of Load Base Split. -aliases: ['/docs/dev/configure-load-base-split/'] --- # Load Base Split diff --git a/configure-memory-usage.md b/configure-memory-usage.md index ebc11d7382155..cf4e13ba672b3 100644 --- a/configure-memory-usage.md +++ b/configure-memory-usage.md @@ -1,7 +1,6 @@ --- title: TiDB Memory Control summary: Learn how to configure the memory quota of a query and avoid OOM (out of memory). -aliases: ['/docs/dev/configure-memory-usage/','/docs/dev/how-to/configure/memory-control/'] --- # TiDB Memory Control @@ -57,7 +56,7 @@ Currently, the memory limit set by `tidb_server_memory_limit` **DOES NOT** termi > > + During the startup process, TiDB does not guarantee that the [`tidb_server_memory_limit`](/system-variables.md#tidb_server_memory_limit-new-in-v640) limit is enforced. If the free memory of the operating system is insufficient, TiDB might still encounter OOM. You need to ensure that the TiDB instance has enough available memory. > + In the process of memory control, the total memory usage of TiDB might slightly exceed the limit set by `tidb_server_memory_limit`. -> + Since v6.5.0, the configruation item `server-memory-quota` is deprecated. To ensure compatibility, after you upgrade your cluster to v6.5.0 or a later version, `tidb_server_memory_limit` will inherit the value of `server-memory-quota`. If you have not configured `server-memory-quota` before the upgrade, the default value of `tidb_server_memory_limit` is used, which is `80%`. +> + Since v6.5.0, the configuration item `server-memory-quota` is deprecated. To ensure compatibility, after you upgrade your cluster to v6.5.0 or a later version, `tidb_server_memory_limit` will inherit the value of `server-memory-quota`. If you have not configured `server-memory-quota` before the upgrade, the default value of `tidb_server_memory_limit` is used, which is `80%`. When the memory usage of a tidb-server instance reaches a certain proportion of the total memory (the proportion is controlled by the system variable [`tidb_server_memory_limit_gc_trigger`](/system-variables.md#tidb_server_memory_limit_gc_trigger-new-in-v640)), tidb-server will try to trigger a Golang GC to relieve memory stress. To avoid frequent GCs that cause performance issues due to the instance memory fluctuating around the threshold, this GC method will trigger GC at most once every minute. diff --git a/configure-placement-rules.md b/configure-placement-rules.md index eb5312c6f115e..790000969f309 100644 --- a/configure-placement-rules.md +++ b/configure-placement-rules.md @@ -1,7 +1,6 @@ --- title: Placement Rules summary: Learn how to configure Placement Rules. -aliases: ['/docs/dev/configure-placement-rules/','/docs/dev/how-to/configure/placement-rules/'] --- # Placement Rules @@ -78,7 +77,7 @@ The Placement Rules feature is enabled by default in v5.0 and later versions of enable-placement-rules = true ``` -In this way, PD enables this feature after the cluster is successfully bootstrapped and generates corresponding rules according to the `max-replicas` and `location-labels` configurations: +In this way, PD enables this feature after the cluster is successfully bootstrapped and generates corresponding rules according to the [`max-replicas`](/pd-configuration-file.md#max-replicas), [`location-labels`](/pd-configuration-file.md#location-labels), and [`isolation-level`](/pd-configuration-file.md#isolation-level) configurations: {{< copyable "" >}} @@ -103,11 +102,12 @@ For a bootstrapped cluster, you can also enable Placement Rules dynamically thro pd-ctl config placement-rules enable ``` -PD also generates default rules based on the `max-replicas` and `location-labels` configurations. +PD also generates default rules based on the `max-replicas`, `location-labels`, and `isolation-level` configurations. > **Note:** > -> After enabling Placement Rules, the previously configured `max-replicas` and `location-labels` no longer take effect. To adjust the replica policy, use the interface related to Placement Rules. +> - When Placement Rules are enabled and multiple rules exist, the previously configured `max-replicas`, `location-labels`, and `isolation-level` no longer take effect. To adjust the replica policy, use the interface related to Placement Rules. +> - When Placement Rules are enabled and only one default rule exists, TiDB will automatically update this default rule when `max-replicas`, `location-labels`, or `isolation-level` configurations are changed. ### Disable Placement Rules @@ -121,7 +121,7 @@ pd-ctl config placement-rules disable > **Note:** > -> After disabling Placement Rules, PD uses the original `max-replicas` and `location-labels` configurations. The modification of rules (when Placement Rules is enabled) will not update these two configurations in real time. In addition, all the rules that have been configured remain in PD and will be used the next time you enable Placement Rules. +> After disabling Placement Rules, PD uses the original `max-replicas`, `location-labels`, and `isolation-level` configurations. The modification of rules (when Placement Rules is enabled) will not update these three configurations in real time. In addition, all the rules that have been configured remain in PD and will be used the next time you enable Placement Rules. ### Set rules using pd-ctl @@ -188,7 +188,9 @@ cat > rules.json < rules.json < **Warning:** +> +> Store limit v2 is an experimental feature. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. + When [`store-limit-version`](/pd-configuration-file.md#store-limit-version-new-in-v710) is set to `v2`, store limit v2 takes effect. In v2 mode, the limit of operators are dynamically adjusted based on the capability of TiKV snapshots. When TiKV has fewer pending tasks, PD increases its scheduling tasks. Otherwise, PD reduces the scheduling tasks for the node. Therefore, you do not need to manually set `store limit` to speed up the scheduling process. In v2 mode, the execution speed of TiKV becomes the main bottleneck during migration. You can check whether the current scheduling speed has reached the upper limit through the **TiKV Details** > **Snapshot** > **Snapshot Speed** panel. To increase or decrease the scheduling speed of a node, you can adjust the TiKV snapshot limit ([`snap-io-max-bytes-per-sec`](/tikv-configuration-file.md#snap-io-max-bytes-per-sec)). diff --git a/configure-time-zone.md b/configure-time-zone.md index 246e226ca16cc..2c4c5028fff96 100644 --- a/configure-time-zone.md +++ b/configure-time-zone.md @@ -1,7 +1,6 @@ --- title: Time Zone Support summary: Learn how to set the time zone and its format. -aliases: ['/docs/dev/configure-time-zone/','/docs/dev/how-to/configure/time-zone/'] --- # Time Zone Support diff --git a/constraints.md b/constraints.md index 097a0d7e277ee..7599f51e083c2 100644 --- a/constraints.md +++ b/constraints.md @@ -1,7 +1,6 @@ --- title: Constraints summary: Learn how SQL Constraints apply to TiDB. -aliases: ['/docs/dev/constraints/','/docs/dev/reference/sql/constraints/'] --- # Constraints diff --git a/control-execution-plan.md b/control-execution-plan.md index cd7a71ddece0e..fd3008dde48c5 100644 --- a/control-execution-plan.md +++ b/control-execution-plan.md @@ -1,5 +1,6 @@ --- title: Control Execution Plan +summary: This chapter introduces methods to control the generation of execution plans in TiDB. It includes using hints, SQL plan management, and the blocklist of optimization rules. Additionally, system variables and the `tidb_opt_fix_control` variable can be modified to control the execution plan. These methods help prevent performance regression caused by behavior changes in the optimizer after cluster upgrades. --- # Control Execution Plan @@ -12,12 +13,12 @@ The first two chapters of SQL Tuning introduce how to understand TiDB's executio -Besides the preceding methods, the execution plan is also affected by some system variables. By modifying these variables at the system level or session level, you can control the generation of the execution plan. Starting from v7.1.0, TiDB introduces a relatively special variable [`tidb_opt_fix_control`](/system-variables.md#tidb_opt_fix_control-new-in-v710). This variable can accept multiple control items to control the behavior of the optimizer in a more fine-grained way, to prevent performance regression caused by behavior changes in the optimizer after cluster upgrade. Refer to [Optimizer Fix Controls](/optimizer-fix-controls.md) for a more detailed introduction. +Besides the preceding methods, the execution plan is also affected by some system variables. By modifying these variables at the system level or session level, you can control the generation of the execution plan. Starting from v6.5.3 and v7.1.0, TiDB introduces a relatively special variable [`tidb_opt_fix_control`](/system-variables.md#tidb_opt_fix_control-new-in-v653-and-v710). This variable can accept multiple control items to control the behavior of the optimizer in a more fine-grained way, to prevent performance regression caused by behavior changes in the optimizer after cluster upgrade. Refer to [Optimizer Fix Controls](/optimizer-fix-controls.md) for a more detailed introduction. -Besides the preceding methods, the execution plan is also affected by some system variables. By modifying these variables at the system level or session level, you can control the generation of the execution plan. Starting from v7.1.0, TiDB introduces a relatively special variable [`tidb_opt_fix_control`](/system-variables.md#tidb_opt_fix_control-new-in-v710). This variable can accept multiple control items to control the behavior of the optimizer in a more fine-grained way, to prevent performance regression caused by behavior changes in the optimizer after cluster upgrade. Refer to [Optimizer Fix Controls](https://docs.pingcap.com/tidb/v7.2/optimizer-fix-controls) for a more detailed introduction. +Besides the preceding methods, the execution plan is also affected by some system variables. By modifying these variables at the system level or session level, you can control the generation of the execution plan. Starting from v6.5.3 and v7.1.0, TiDB introduces a relatively special variable [`tidb_opt_fix_control`](/system-variables.md#tidb_opt_fix_control-new-in-v653-and-v710). This variable can accept multiple control items to control the behavior of the optimizer in a more fine-grained way, to prevent performance regression caused by behavior changes in the optimizer after cluster upgrade. Refer to [Optimizer Fix Controls](https://docs.pingcap.com/tidb/v7.2/optimizer-fix-controls) for a more detailed introduction. diff --git a/coprocessor-cache.md b/coprocessor-cache.md index 8c995ed0648c1..c386105fc863a 100644 --- a/coprocessor-cache.md +++ b/coprocessor-cache.md @@ -1,7 +1,6 @@ --- title: Coprocessor Cache summary: Learn the features of Coprocessor Cache. -aliases: ['/docs/dev/coprocessor-cache/'] --- # Coprocessor Cache diff --git a/daily-check.md b/daily-check.md index 6d82b984dd4c9..02b4565724ebe 100644 --- a/daily-check.md +++ b/daily-check.md @@ -1,7 +1,6 @@ --- title: Daily Check summary: Learn about performance indicators of the TiDB cluster. -aliases: ['/docs/dev/daily-check/'] --- # Daily Check @@ -38,14 +37,17 @@ You can locate the slow SQL statement executed in the cluster. Then you can opti ![Region panel](/media/region-panel.png) -+ `miss-peer-region-count`: The number of Regions without enough replicas. This value is not always greater than `0`. ++ `down-peer-region-count`: The number of Regions with an unresponsive peer reported by the Raft leader. ++ `empty-region-count`: The number of empty Regions, with a size of smaller than 1 MiB. These Regions are generated by executing the `TRUNCATE TABLE`/`DROP TABLE` statement. If this number is large, you can consider enabling `Region Merge` to merge Regions across tables. + `extra-peer-region-count`: The number of Regions with extra replicas. These Regions are generated during the scheduling process. -+ `empty-region-count`: The number of empty Regions, generated by executing the `TRUNCATE TABLE`/`DROP TABLE` statement. If this number is large, you can consider enabling `Region Merge` to merge Regions across tables. ++ `learner-peer-region-count`: The number of Regions with the learner peer. The sources of learner peers can be various, for example, the learner peers in TiFlash, and the learner peers included in the configured Placement Rules. ++ `miss-peer-region-count`: The number of Regions without enough replicas. This value is not always greater than `0`. ++ `offline-peer-region-count`: The number of Regions during the peer offline process. ++ `oversized-region-count`: The number of Regions with a size larger than `region-max-size` or `region-max-keys`. + `pending-peer-region-count`: The number of Regions with outdated Raft logs. It is normal that a few pending peers are generated in the scheduling process. However, it is not normal if this value is large for a period of time (longer than 30 minutes). -+ `down-peer-region-count`: The number of Regions with an unresponsive peer reported by the Raft leader. -+ `offline-peer-region-count`: The number of Regions during the offline process. ++ `undersized-region-count`: The number of Regions with a size smaller than `max-merge-region-size` or `max-merge-region-keys`. -Generally, it is normal that these values are not `0`. However, it is not normal that they are not `0` for quite a long time. +Generally, it is normal for these metrics to show small and non-zero values. ### KV Request Duration diff --git a/dashboard/continuous-profiling.md b/dashboard/continuous-profiling.md index 4b25399342dca..e68e7ac09af14 100644 --- a/dashboard/continuous-profiling.md +++ b/dashboard/continuous-profiling.md @@ -1,6 +1,6 @@ --- title: TiDB Dashboard Instance Profiling - Continuous Profiling -summary: Learn how to collect performance data from TiDB, TiKV and PD continuously to reduce MTTR. +summary: TiDB Dashboard Continuous Profiling allows experts to collect and analyze performance data continuously from each instance, helping to pinpoint and resolve sophisticated performance problems. It stores more data than Manual Profiling, enabling analysis of both current and historical issues. The feature can be accessed through the dashboard or a browser, and can be enabled and disabled as needed. Performance impact is minimal, making it suitable for production environments. --- # TiDB Dashboard Instance Profiling - Continuous Profiling @@ -28,7 +28,7 @@ All performance data in [Manual Profiling](/dashboard/dashboard-profiling.md#sup - CPU: The CPU overhead of each internal function on TiDB, TiKV, TiFlash, and PD instances -- Heap: The memory consumption of each internal function on TiDB and PD instances +- Heap: The memory consumption of each internal function on TiDB, TiKV, and PD instances - Mutex: The mutex contention states on TiDB and PD instances diff --git a/dashboard/dashboard-access.md b/dashboard/dashboard-access.md index e3e59ecb6c99a..4de76d7cf68cb 100644 --- a/dashboard/dashboard-access.md +++ b/dashboard/dashboard-access.md @@ -1,7 +1,6 @@ --- title: Access TiDB Dashboard -summary: Learn how to access TiDB Dashboard. -aliases: ['/docs/dev/dashboard/dashboard-access/'] +summary: To access TiDB Dashboard, visit the specified URL in your browser. For multiple PD instances, replace the address with any PD instance address and port. Use Chrome, Firefox, or Edge browsers of newer versions. Sign in with the TiDB root account or a user-defined SQL user. The session remains valid for 24 hours. Switch between English and Chinese languages. To log out, click the user name and then the Logout button. --- # Access TiDB Dashboard @@ -10,7 +9,7 @@ To access TiDB Dashboard, visit via your brows > **Note:** > -> TiDB v6.5.0 (and later) and TiDB Operator v1.4.0 (and later) support deploying TiDB Dashboard as an independent Pod on Kubernetes. Using TiDB Operator, you can access the IP address of this Pod to start TiDB Dashboard. For details, see [Deploy TiDB Dashboard independently in TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/dev/get-started#deploy-tidb-dashboard-independently). +> TiDB v6.5.0 (and later) and TiDB Operator v1.4.0 (and later) support deploying TiDB Dashboard as an independent Pod on Kubernetes. Using TiDB Operator, you can access the IP address of this Pod to start TiDB Dashboard. For details, see [Deploy TiDB Dashboard independently in TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/v1.5/get-started#deploy-tidb-dashboard-independently). ## Access TiDB Dashboard when multiple PD instances are deployed diff --git a/dashboard/dashboard-cluster-info.md b/dashboard/dashboard-cluster-info.md index e8df69f6b9095..968f47b7acd27 100644 --- a/dashboard/dashboard-cluster-info.md +++ b/dashboard/dashboard-cluster-info.md @@ -1,7 +1,6 @@ --- title: TiDB Dashboard Cluster Information Page -summary: View the running status of TiDB, TiKV, PD, TiFlash components in the entire cluster and the running status of the host on which these components are located. -aliases: ['/docs/dev/dashboard/dashboard-cluster-info/'] +summary: The TiDB Dashboard Cluster Information Page allows users to view the running status of TiDB, TiKV, PD, and TiFlash components in the entire cluster, as well as the running status of the host on which these components are located. Users can access the page by logging in to TiDB Dashboard and clicking on Cluster Info in the left navigation menu, or by visiting a specific URL in their browser. The page provides instance, host, and disk lists, showing detailed information about each component and its running status. --- # TiDB Dashboard Cluster Information Page diff --git a/dashboard/dashboard-diagnostics-access.md b/dashboard/dashboard-diagnostics-access.md index c285c2770c37a..b7ed53b4e6089 100644 --- a/dashboard/dashboard-diagnostics-access.md +++ b/dashboard/dashboard-diagnostics-access.md @@ -1,7 +1,6 @@ --- title: TiDB Dashboard Cluster Diagnostic Page -summary: Learn how to use the cluster diagnostic page. -aliases: ['/docs/dev/dashboard/dashboard-diagnostics-access/'] +summary: TiDB Dashboard Cluster Diagnostics diagnoses cluster problems and summarizes results into a web page. Access the page through the dashboard or browser. Generate diagnostic and comparison reports for specified time ranges. Historical reports are also available. --- # TiDB Dashboard Cluster Diagnostics Page diff --git a/dashboard/dashboard-diagnostics-report.md b/dashboard/dashboard-diagnostics-report.md index bd334978aa97c..83a20e5684191 100644 --- a/dashboard/dashboard-diagnostics-report.md +++ b/dashboard/dashboard-diagnostics-report.md @@ -1,7 +1,6 @@ --- title: TiDB Dashboard Diagnostic Report -summary: Learn the TiDB Dashboard diagnostic report. -aliases: ['/docs/dev/dashboard/dashboard-diagnostics-report/'] +summary: TiDB Dashboard Diagnostic Report introduces diagnostic report content, including basic, diagnostic, load, overview, monitoring, and configuration information. It also includes comparison report details, DIFF_RATIO explanation, and Maximum Different Item table. --- # TiDB Dashboard Diagnostic Report diff --git a/dashboard/dashboard-diagnostics-usage.md b/dashboard/dashboard-diagnostics-usage.md index 918270171d24a..bdb139d15fdc5 100644 --- a/dashboard/dashboard-diagnostics-usage.md +++ b/dashboard/dashboard-diagnostics-usage.md @@ -1,7 +1,6 @@ --- title: Locate Problems Using Diagnostic Report of TiDB Dashboard -summary: Learn how to locate problems using diagnostic report of TiDB Dashboard. -aliases: ['/docs/dev/dashboard/dashboard-diagnostics-usage/'] +summary: TiDB Dashboard's diagnostic report helps locate problems by comparing system performance at different time ranges. It identifies issues like QPS decrease, latency increase, and slow queries, providing detailed analysis and SQL statements for further investigation. This comparison report is essential for quickly identifying and addressing performance issues. --- # Locate Problems Using Diagnostic Report of TiDB Dashboard diff --git a/dashboard/dashboard-faq.md b/dashboard/dashboard-faq.md index 05d9d97fd4ce5..2aefc70e2693b 100644 --- a/dashboard/dashboard-faq.md +++ b/dashboard/dashboard-faq.md @@ -1,7 +1,6 @@ --- title: TiDB Dashboard FAQs -summary: Learn about the frequently asked questions (FAQs) and answers about TiDB Dashboard. -aliases: ['/docs/dev/dashboard/dashboard-faq/'] +summary: This document summarizes FAQs about TiDB Dashboard. It covers access-related, UI-related, and deployment issues, providing solutions for each problem. If further assistance is needed, support can be obtained from PingCAP or the community. --- # TiDB Dashboard FAQs @@ -61,7 +60,7 @@ The possible reason is that you have enabled the Prepared Plan Cache feature of ### A `required component NgMonitoring is not started` error is shown -NgMonitoring is an advanced monitoring component built in TiDB clusters of v5.4.0 and later versions to support TiDB Dashboard features such as **Continuous Profiling** and **Top SQL**. NgMonitoring is automatically deployed when you deploy or upgrade a cluster with a newer version of TiUP. For clusters deployed using TiDB Operator, you can deploy NgMonitoring manually by referring to [Enable Continuous Profiling](https://docs.pingcap.com/tidb-in-kubernetes/dev/access-dashboard/#enable-continuous-profiling). +NgMonitoring is an advanced monitoring component built in TiDB clusters of v5.4.0 and later versions to support TiDB Dashboard features such as **Continuous Profiling** and **Top SQL**. NgMonitoring is automatically deployed when you deploy or upgrade a cluster with a newer version of TiUP. For clusters deployed using TiDB Operator, you can deploy NgMonitoring manually by referring to [Enable Continuous Profiling](https://docs.pingcap.com/tidb-in-kubernetes/v1.5/access-dashboard/#enable-continuous-profiling). If the web page shows `required component NgMonitoring is not started`, you can troubleshoot the deployment issue as follows: @@ -127,7 +126,7 @@ If the error message is still prompted after performing steps above, [get suppor
Clusters Deployed using TiDB Operator -Deploy the NgMonitoring component by following instructions in the [Enable Continuous Profiling](https://docs.pingcap.com/tidb-in-kubernetes/dev/access-dashboard/#enable-continuous-profiling) section in TiDB Operator documentation. +Deploy the NgMonitoring component by following instructions in the [Enable Continuous Profiling](https://docs.pingcap.com/tidb-in-kubernetes/v1.5/access-dashboard/#enable-continuous-profiling) section in TiDB Operator documentation.
diff --git a/dashboard/dashboard-intro.md b/dashboard/dashboard-intro.md index a207daa50d315..ce4d93a783ca1 100644 --- a/dashboard/dashboard-intro.md +++ b/dashboard/dashboard-intro.md @@ -1,7 +1,6 @@ --- title: TiDB Dashboard Introduction -summary: Introduce TiDB Dashboard. -aliases: ['/docs/dev/dashboard/dashboard-intro/'] +summary: TiDB Dashboard is a Web UI for monitoring, diagnosing, and managing the TiDB cluster. It shows overall running status, component and host status, traffic distribution, SQL statement execution information, slow queries, cluster diagnostics, log search, resource control, and profiling data collection. --- # TiDB Dashboard Introduction @@ -10,7 +9,7 @@ TiDB Dashboard is a Web UI for monitoring, diagnosing, and managing the TiDB clu > **Note:** > -> TiDB v6.5.0 (and later) and TiDB Operator v1.4.0 (and later) support deploying TiDB Dashboard as an independent Pod on Kubernetes. For details, see [Deploy TiDB Dashboard independently in TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/dev/get-started#deploy-tidb-dashboard-independently). +> TiDB v6.5.0 (and later) and TiDB Operator v1.4.0 (and later) support deploying TiDB Dashboard as an independent Pod on Kubernetes. For details, see [Deploy TiDB Dashboard independently in TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/v1.5/get-started#deploy-tidb-dashboard-independently). ![TiDB Dashboard interface](/media/dashboard/dashboard-intro.gif) diff --git a/dashboard/dashboard-key-visualizer.md b/dashboard/dashboard-key-visualizer.md index 9ee08da701000..e31c884a29059 100644 --- a/dashboard/dashboard-key-visualizer.md +++ b/dashboard/dashboard-key-visualizer.md @@ -1,7 +1,6 @@ --- title: Key Visualizer Page -summary: Learn how to use Key Visualizer to monitor traffic. -aliases: ['/docs/dev/dashboard/dashboard-key-visualizer/','/docs/dev/key-visualizer-monitoring-tool/'] +summary: TiDB Dashboard's Key Visualizer page analyzes and troubleshoots traffic hotspots in the TiDB cluster. It visually shows traffic changes over time, and allows users to zoom in on specific time periods or region ranges. The page also provides settings to adjust brightness, select metrics, and refresh the heatmap. It identifies common heatmap types and offers solutions to address hotspot issues. --- # Key Visualizer Page @@ -38,7 +37,7 @@ This section introduces the basic concepts that relate to Key Visualizer. In a TiDB cluster, the stored data is distributed among TiKV instances. Logically, TiKV is a huge and orderly key-value map. The whole key-value space is divided into many segments and each segment consists of a series of adjacent keys. Such segment is called a `Region`. -For detailed introduction of Region, refer to [TiDB Internal (I) - Data Storage](https://en.pingcap.com/blog/tidb-internal-data-storage/). +For detailed introduction of Region, refer to [TiDB Internal (I) - Data Storage](https://www.pingcap.com/blog/tidb-internal-data-storage/). ### Hotspot diff --git a/dashboard/dashboard-log-search.md b/dashboard/dashboard-log-search.md index 713955090c764..4a32139f7a423 100644 --- a/dashboard/dashboard-log-search.md +++ b/dashboard/dashboard-log-search.md @@ -1,7 +1,6 @@ --- title: TiDB Dashboard Log Search Page -summary: Learn how to search logs of all nodes using the log search page of TiDB Dashboard. -aliases: ['/docs/dev/dashboard/dashboard-log-search/'] +summary: TiDB Dashboard log search page allows users to search logs, preview results, and download logs. Users can access the page after logging in, and specify time range, log level, keywords, and components for the search. The search result page displays parameter options, search progress, and search results. Users can download selected logs, cancel running tasks, and retry failed tasks. The search history list shows details of past searches and allows users to delete unnecessary history. --- # TiDB Dashboard Log Search Page diff --git a/dashboard/dashboard-metrics-relation.md b/dashboard/dashboard-metrics-relation.md index 19640d50851d1..25425bee92927 100644 --- a/dashboard/dashboard-metrics-relation.md +++ b/dashboard/dashboard-metrics-relation.md @@ -1,6 +1,6 @@ --- title: TiDB Dashboard Metrics Relation Graph -summary: Learn TiDB Dashboard metrics relation graph. +summary: TiDB Dashboard introduces a feature called metrics relation graph, which helps users understand the duration of each internal process in a TiDB cluster. After logging in, users can access the graph and see the proportion of each monitoring metric's duration to the total query duration. Each box area represents a monitoring metric and provides information such as the total duration and proportion to the total query duration. The graph also illustrates the parent-child relations between nodes, helping users understand the relations of each monitoring metric. --- # TiDB Dashboard Metrics Relation Graph diff --git a/dashboard/dashboard-monitoring.md b/dashboard/dashboard-monitoring.md index 85a00bc202e7d..eb9e0595a808d 100644 --- a/dashboard/dashboard-monitoring.md +++ b/dashboard/dashboard-monitoring.md @@ -1,6 +1,6 @@ --- title: TiDB Dashboard Monitoring Page -summary: Learn how to view the Performance Overview dashboard on TiDB Dashboard and understand key metrics displayed on the Performance Overview dashboard. +summary: The TiDB Dashboard Monitoring Page allows users to analyze performance efficiently and identify database bottlenecks. Key metrics include database time, SQL execution time, QPS, connection count, TiDB and TiKV CPU, duration, connection idle duration, parse, compile, and execute duration, TiDB KV request duration, TiKV gRPC duration, PD TSO wait/RPC duration, storage async write duration, store duration, apply duration, append log duration, commit log duration, and apply log duration. --- # TiDB Dashboard Monitoring Page @@ -69,11 +69,11 @@ Number of queries using plan cache per second in all TiDB instances ### KV/TSO Request OPS - kv request total: Total number of KV requests per second in all TiDB instances -- kv request by type: Number of KV requests per second in all TiDB instances based on such types as `Get`, `Prewrite`, and `Commit`. -- tso - cmd: Number of `tso cmd` requests per second in all TiDB instances -- tso - request: Number of `tso request` requests per second in all TiDB instances +- kv request by type: Number of KV requests per second in all TiDB instances based on such types as `Get`, `Prewrite`, and `Commit` +- tso - cmd: Number of gRPC requests per second that TiDB sends to PD in all TiDB instances; each gRPC request contains a batch of TSO requests +- tso - request: Number of TSO requests per second in all TiDB instances -Generally, dividing `tso - cmd` by `tso - request` yields the average batch size of requests per second. +Generally, `tso - request` divided by `tso - cmd` is the average size of TSO request batches per second. ### Connection Count diff --git a/dashboard/dashboard-ops-deploy.md b/dashboard/dashboard-ops-deploy.md index cf7939aea9fb9..b668c287187e8 100644 --- a/dashboard/dashboard-ops-deploy.md +++ b/dashboard/dashboard-ops-deploy.md @@ -1,7 +1,6 @@ --- title: Deploy TiDB Dashboard -summary: Learn how to deploy TiDB Dashboard. -aliases: ['/docs/dev/dashboard/dashboard-ops-deploy/'] +summary: TiDB Dashboard is built into PD for v4.0 or higher. No additional deployment is needed. It can also be deployed independently on Kubernetes. When multiple PD instances are deployed, only one serves the Dashboard. Use `tiup cluster display` to check the serving instance. You can disable and re-enable the Dashboard using `tiup ctl`. --- # Deploy TiDB Dashboard @@ -10,7 +9,7 @@ The TiDB Dashboard UI is built into the PD component for v4.0 or higher versions > **Note:** > -> TiDB v6.5.0 (and later) and TiDB Operator v1.4.0 (and later) support deploying TiDB Dashboard as an independent Pod on Kubernetes. For details, see [Deploy TiDB Dashboard independently in TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/dev/get-started#deploy-tidb-dashboard-independently). +> TiDB v6.5.0 (and later) and TiDB Operator v1.4.0 (and later) support deploying TiDB Dashboard as an independent Pod on Kubernetes. For details, see [Deploy TiDB Dashboard independently in TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/v1.5/get-started#deploy-tidb-dashboard-independently). See the following documents to learn how to deploy a standard TiDB cluster: diff --git a/dashboard/dashboard-ops-reverse-proxy.md b/dashboard/dashboard-ops-reverse-proxy.md index fb417e2e4d363..bc12148875168 100644 --- a/dashboard/dashboard-ops-reverse-proxy.md +++ b/dashboard/dashboard-ops-reverse-proxy.md @@ -1,6 +1,6 @@ --- title: Use TiDB Dashboard behind a Reverse Proxy -aliases: ['/docs/dev/dashboard/dashboard-ops-reverse-proxy/'] +summary: TiDB Dashboard can be safely exposed using a reverse proxy. To do this, get the actual TiDB Dashboard address and configure the reverse proxy using either HAProxy or NGINX. You can also customize the path prefix for the TiDB Dashboard service. To enhance security, consider configuring a firewall. --- # Use TiDB Dashboard behind a Reverse Proxy diff --git a/dashboard/dashboard-ops-security.md b/dashboard/dashboard-ops-security.md index d3f491e80d745..1621419b7435b 100644 --- a/dashboard/dashboard-ops-security.md +++ b/dashboard/dashboard-ops-security.md @@ -1,7 +1,6 @@ --- title: Secure TiDB Dashboard -summary: Learn how to improve the security of TiDB Dashboard. -aliases: ['/docs/dev/dashboard/dashboard-ops-security/'] +summary: TiDB Dashboard requires enhanced security measures, including setting a strong password for the root user, creating a least-privileged user, and using a firewall to block untrusted access. It is also recommended to use a reverse proxy and enable TLS for further security. --- # Secure TiDB Dashboard @@ -26,7 +25,7 @@ It is recommended that you create a least-privileged SQL user to access and sign > **Note:** > -> TiDB v6.5.0 (and later) and TiDB Operator v1.4.0 (and later) support deploying TiDB Dashboard as an independent Pod on Kubernetes. Using TiDB Operator, you can access the IP address of this Pod to start TiDB Dashboard. This port does not communicate with other privileged interfaces of PD and no extra firewall is required if provided externally. For details, see [Deploy TiDB Dashboard independently in TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/dev/get-started#deploy-tidb-dashboard-independently). +> TiDB v6.5.0 (and later) and TiDB Operator v1.4.0 (and later) support deploying TiDB Dashboard as an independent Pod on Kubernetes. Using TiDB Operator, you can access the IP address of this Pod to start TiDB Dashboard. This port does not communicate with other privileged interfaces of PD and no extra firewall is required if provided externally. For details, see [Deploy TiDB Dashboard independently in TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/v1.5/get-started#deploy-tidb-dashboard-independently). TiDB Dashboard provides services through the PD client port, which defaults to . Although TiDB Dashboard requires identity authentication, other privileged interfaces (such as ) in PD carried on the PD client port do not require identity authentication and can perform privileged operations. Therefore, exposing the PD client port directly to the external network is extremely risky. diff --git a/dashboard/dashboard-overview.md b/dashboard/dashboard-overview.md index 075c98fdf99a4..fd7c07a534cf1 100644 --- a/dashboard/dashboard-overview.md +++ b/dashboard/dashboard-overview.md @@ -1,7 +1,6 @@ --- title: Overview Page -summary: Learn the overview page of TiDB Dashboard. -aliases: ['/docs/dev/dashboard/dashboard-overview/'] +summary: The TiDB overview page displays cluster QPS, latency, top SQL statements, recent slow queries, instance status, and monitor/alert links. Access it via TiDB Dashboard or left navigation menu. QPS and latency require Prometheus monitoring. Top SQL and slow queries need SQL Statements and slow query logs enabled. Instance status shows total and abnormal instances. Monitor and alert links lead to Grafana dashboard, AlertManager, and cluster diagnostics. --- # Overview Page diff --git a/dashboard/dashboard-profiling.md b/dashboard/dashboard-profiling.md index e4aa12c12dfef..20f9e9514ff28 100644 --- a/dashboard/dashboard-profiling.md +++ b/dashboard/dashboard-profiling.md @@ -1,7 +1,6 @@ --- title: TiDB Dashboard Instance Profiling - Manual Profiling -summary: Learn how to collect performance data to analyze sophisticated problems. -aliases: ['/docs/dev/dashboard/dashboard-profiling/'] +summary: Manual Profiling allows users to collect current performance data on demand for TiDB, TiKV, PD, and TiFlash instances. Experts can analyze resource consumption details like CPU and memory to pinpoint ongoing performance problems. Access the page through TiDB Dashboard or a browser. Start profiling by choosing target instances and modify the duration if needed. View real-time progress and download performance data after profiling is completed. View profiling history for detailed operations. --- # TiDB Dashboard Instance Profiling - Manual Profiling @@ -24,7 +23,9 @@ The following performance data are currently supported: > The CPU overhead of TiKV and TiFlash instances is currently not supported in ARM architecture. -- Heap: The memory consumption of each internal function on TiDB and PD instances +- Heap: The memory consumption of each internal function on TiDB, TiKV, and PD instances + + > Starting from v7.5, TiDB supports the TiKV Heap Profile. The Perl dependency is required in the running environment of TiDB Dashboard. Otherwise an error will occur. - Mutex: The mutex contention states on TiDB and PD instances diff --git a/dashboard/dashboard-resource-manager.md b/dashboard/dashboard-resource-manager.md index d2abddcf101c3..19c48bd94377e 100644 --- a/dashboard/dashboard-resource-manager.md +++ b/dashboard/dashboard-resource-manager.md @@ -1,6 +1,6 @@ --- title: TiDB Dashboard Resource Manager Page -summary: Introduce how to use the Resource Manager page in TiDB Dashboard to view the information about resource control, so you can estimate cluster capacity before resource planning and allocate resources more effectively. +summary: TiDB Dashboard Resource Manager Page helps cluster administrators implement resource isolation by creating resource groups and setting quotas. It provides methods to estimate cluster capacity and monitor resource consumption. Access the page through TiDB Dashboard or a browser. The page includes sections for configuration, capacity estimation, and metrics. Capacity estimation methods include hardware deployment and actual workload. Monitoring metrics include total RU consumed, RU consumed by resource groups, TiDB CPU quota and usage, TiKV CPU quota and usage, and TiKV IO MBps. --- # TiDB Dashboard Resource Manager Page @@ -47,7 +47,7 @@ Before resource planning, you need to know the overall capacity of the cluster. ![Calibrate by Hardware](/media/dashboard/dashboard-resource-manager-calibrate-by-hardware.png) - The **Total RU of user resource groups** represents the total amount of RU for all user resource groups, excluding the `default` resource group. If this value is less than the estimated capacity, the system triggers an alert. By default, the system allocates unlimited usage to the predefined `default` resource group. When all users belong to the `default` resource group, resources are allocated in the same way as when resource control is disabled. + The **Total RU of user resource groups** represents the total amount of RU for all user resource groups, excluding the `default` resource group. If this value is more than the estimated capacity, the system triggers an alert. By default, the system allocates unlimited usage to the predefined `default` resource group. When all users belong to the `default` resource group, resources are allocated in the same way as when resource control is disabled. - [Estimate capacity based on actual workload](/sql-statements/sql-statement-calibrate-resource.md#estimate-capacity-based-on-actual-workload) @@ -63,6 +63,10 @@ Before resource planning, you need to know the overall capacity of the cluster. You can select an appropriate time range using **CPU Usage** in the [Metrics](#metrics) section. +> **Note:** +> +> To use the capacity estimation feature, the current login user must have the `SUPER` or `RESOURCE_GROUP_ADMIN` privilege and the `SELECT` privilege for some system tables. Before using this feature, ensure the current user has these privileges. Otherwise, some features might not work properly. For more information, see [`CALIBRATE RESOURCE`](/sql-statements/sql-statement-calibrate-resource.md#privileges). + ## Metrics By observing the metrics on the panels, you can understand the current overall resource consumption status of the cluster. The monitoring metrics and their meanings are as follows: @@ -75,4 +79,4 @@ By observing the metrics on the panels, you can understand the current overall r - TiKV - CPU Quota: The maximum CPU usage of TiKV. - CPU Usage: The total CPU usage of all TiKV instances. - - IO MBps: The total I/O throughput of all TiKV instances. \ No newline at end of file + - IO MBps: The total I/O throughput of all TiKV instances. diff --git a/dashboard/dashboard-session-share.md b/dashboard/dashboard-session-share.md index dfd4ddb646980..b0b7e6fc1887b 100644 --- a/dashboard/dashboard-session-share.md +++ b/dashboard/dashboard-session-share.md @@ -1,6 +1,6 @@ --- title: Share TiDB Dashboard Sessions -summary: Learn how to share the current TiDB Dashboard session to other users. +summary: TiDB Dashboard allows users to share their current session with others, eliminating the need for a user password. The inviter can generate an authorization code with specific sharing settings and provide it to the invitee. The invitee can then use the authorization code to sign in without a password. --- # Share TiDB Dashboard Sessions diff --git a/dashboard/dashboard-session-sso.md b/dashboard/dashboard-session-sso.md index 2a46b56538374..900c1944824d7 100644 --- a/dashboard/dashboard-session-sso.md +++ b/dashboard/dashboard-session-sso.md @@ -1,6 +1,6 @@ --- title: Configure SSO for TiDB Dashboard -summary: Learn how to enable SSO to sign into TiDB Dashboard. +summary: TiDB Dashboard supports OIDC-based SSO for sign-in authentication. To enable SSO, fill OIDC Client ID and Discovery URL, authorize impersonation, and save the configuration. To disable SSO, deselect the option and update the configuration. If the SQL user password changes, re-enter it to enable SSO again. After configuration, sign in via SSO by clicking "Sign in via Company Account" and completing the sign-in process. Examples of using Okta, Auth0, and Casdoor for SSO configuration are provided. --- # Configure SSO for TiDB Dashboard @@ -104,7 +104,7 @@ First, create an Okta Application Integration to integrate SSO. ![Sample Step](/media/dashboard/dashboard-session-sso-okta-1.png) -4. In the poped up dialog, choose **OIDC - OpenID Connect** in **Sign-in method**. +4. In the popped up dialog, choose **OIDC - OpenID Connect** in **Sign-in method**. 5. Choose **Single-Page Application** in **Application Type**. diff --git a/dashboard/dashboard-slow-query.md b/dashboard/dashboard-slow-query.md index 4c400000c52b0..40fb09c82adb4 100644 --- a/dashboard/dashboard-slow-query.md +++ b/dashboard/dashboard-slow-query.md @@ -1,7 +1,6 @@ --- title: Slow Queries Page of TiDB Dashboard -summary: Learn the Slow Queries page of TiDB Dashboard. -aliases: ['/docs/dev/dashboard/dashboard-slow-query/'] +summary: TiDB Dashboard's Slow Queries page allows users to search and view slow queries in the cluster. Queries with an execution time over 300 milliseconds are considered slow. Users can adjust the threshold and access the page through the dashboard or a browser. They can also change filters, display more columns, export queries, and view execution details. --- # Slow Queries Page of TiDB Dashboard @@ -60,6 +59,10 @@ Click any item in the list to display detailed execution information of the slow ### SQL +> **Note:** +> +> The maximum length of the query recorded in the `Query` column is limited by the [`tidb_stmt_summary_max_sql_length`](/system-variables.md#tidb_stmt_summary_max_sql_length-new-in-v40) system variable. + Click the **Expand** button to view the detailed information of an item. Click the **Copy** button to copy the detailed information to the clipboard. ### Execution plans diff --git a/dashboard/dashboard-statement-details.md b/dashboard/dashboard-statement-details.md index 02c8bc1b9d37a..7bd8040836d31 100644 --- a/dashboard/dashboard-statement-details.md +++ b/dashboard/dashboard-statement-details.md @@ -1,7 +1,6 @@ --- title: Statement Execution Details of TiDB Dashboard -summary: View the execution details of a single SQL statement in TiDB Dashboard. -aliases: ['/docs/dev/dashboard/dashboard-statement-details/'] +summary: TiDB Dashboard provides detailed information on SQL statement execution, including SQL template overview, execution plan list, and plan binding feature. Starting from v6.6.0, fast plan binding allows quick binding and dropping of execution plans. However, it has limitations and requires SUPER privilege. The execution detail of plans includes SQL sample, complete execution plan information, and basic execution details. Visual representations of execution plans are available in table, text, and graph formats. Additional tabs provide information on execution time, Coprocessor read, transaction, and slow queries. --- # Statement Execution Details of TiDB Dashboard diff --git a/dashboard/dashboard-statement-list.md b/dashboard/dashboard-statement-list.md index 3ffe33677f252..1f56ac64d423f 100644 --- a/dashboard/dashboard-statement-list.md +++ b/dashboard/dashboard-statement-list.md @@ -1,7 +1,6 @@ --- title: SQL Statements Page of TiDB Dashboard -summary: View the execution status of all SQL statements in the TiDB cluster. -aliases: ['/docs/dev/dashboard/dashboard-statement-list/'] +summary: The SQL statements page in TiDB Dashboard shows the execution status of all SQL statements in the cluster. It allows users to analyze long-running SQL statements and provides options to access, filter, display more columns, sort, and change settings. The page also includes a feature to limit the number of stored SQL statements. For more details, visit the TiDB Dashboard documentation. --- # SQL Statements Page of TiDB Dashboard @@ -22,7 +21,7 @@ All the data shown on the SQL statement summary page are from the TiDB statement > **Note:** > -> In the **Mean Latency** column of the SQL statement summary page, the blue bar indicates the average execution time. If there is a yellow line on the blue bar for an SQL statement, the left and right sides of the yellow line respectively represent the minimum and maximum execution time of the SQL statement during the recent data collection cycle. +> In the **Mean Latency** column of the SQL statement summary page, the blue bar indicates the average execution time. If there is a yellow line on the blue bar for an SQL statement, the left and right sides of the yellow line respectively represent the minimum and maximum execution time of the SQL statement during the recent data collection cycle. ### Change Filters diff --git a/dashboard/dashboard-user.md b/dashboard/dashboard-user.md index 025efe82d0b42..f5c2963d718fc 100644 --- a/dashboard/dashboard-user.md +++ b/dashboard/dashboard-user.md @@ -1,7 +1,6 @@ --- title: TiDB Dashboard User Management -summary: Learn how to create SQL users to access TiDB Dashboard. -aliases: ['/docs/dev/dashboard/dashboard-user/'] +summary: TiDB Dashboard uses the same user privilege system as TiDB. SQL users need specific privileges to access the dashboard, including PROCESS, SHOW DATABASES, CONFIG, DASHBOARD_CLIENT, and more. It's recommended to create users with only the required privileges to prevent unintended operations. Users with high privileges can also sign in. To create a least-privileged SQL user, grant the necessary privileges and use role-based access control (RBAC) if needed. --- # TiDB Dashboard User Management @@ -58,8 +57,8 @@ If an SQL user does not meet the preceding privilege requirements, the user fail -- To modify the configuration items on the interface after signing in to TiDB Dashboard, the user-defined SQL user must be granted with the following privilege. GRANT SYSTEM_VARIABLES_ADMIN ON *.* TO 'dashboardAdmin'@'%'; - - -- To use the Fast Bind Executions Plan feature (https://docs.pingcap.com/tidb/dev/dashboard-statement-details#fast-plan-binding) on the interface after signing in to TiDB Dashboard, the user-defined SQL user must be granted with the following privileges. + + -- To use the Fast Bind Executions Plan feature (https://docs.pingcap.com/tidb/v7.5/dashboard-statement-details#fast-plan-binding) on the interface after signing in to TiDB Dashboard, the user-defined SQL user must be granted with the following privileges. GRANT SYSTEM_VARIABLES_ADMIN ON *.* TO 'dashboardAdmin'@'%'; GRANT SUPER ON *.* TO 'dashboardAdmin'@'%'; ``` @@ -77,8 +76,8 @@ If an SQL user does not meet the preceding privilege requirements, the user fail -- To modify the configuration items on the interface after signing in to TiDB Dashboard, the user-defined SQL user must be granted with the following privilege. GRANT SYSTEM_VARIABLES_ADMIN ON *.* TO 'dashboardAdmin'@'%'; - - -- To use the Fast Bind Executions Plan feature (https://docs.pingcap.com/tidb/dev/dashboard-statement-details#fast-plan-binding) on the interface after signing in to TiDB Dashboard, the user-defined SQL user must be granted with the following privileges. + + -- To use the Fast Bind Executions Plan feature (https://docs.pingcap.com/tidb/v7.5/dashboard-statement-details#fast-plan-binding) on the interface after signing in to TiDB Dashboard, the user-defined SQL user must be granted with the following privileges. GRANT SYSTEM_VARIABLES_ADMIN ON *.* TO 'dashboardAdmin'@'%'; GRANT SUPER ON *.* TO 'dashboardAdmin'@'%'; ``` @@ -95,7 +94,7 @@ The following example demonstrates how to create a role and a user to access TiD GRANT SHOW DATABASES ON *.* TO 'dashboard_access'@'%'; GRANT DASHBOARD_CLIENT ON *.* TO 'dashboard_access'@'%'; GRANT SYSTEM_VARIABLES_ADMIN ON *.* TO 'dashboard_access'@'%'; - GRANT SUPER ON *.* TO 'dashboardAdmin'@'%'; + GRANT SUPER ON *.* TO 'dashboardAdmin'@'%'; ``` 2. Grant the `dashboard_access` role to other users and set `dashboard_access` as the default role: diff --git a/dashboard/top-sql.md b/dashboard/top-sql.md index f6f0cfc6fd48b..a1293519daa9c 100644 --- a/dashboard/top-sql.md +++ b/dashboard/top-sql.md @@ -1,6 +1,6 @@ --- title: TiDB Dashboard Top SQL page -summary: Learn how to use Top SQL to find SQL statements with high CPU overhead. +summary: TiDB Dashboard Top SQL allows real-time monitoring and visualization of CPU overhead for SQL statements in your database. It helps optimize performance by identifying high CPU load statements and provides detailed execution information. It's suitable for analyzing performance issues and can be accessed through TiDB Dashboard or a browser. The feature has a slight impact on cluster performance and is now generally available for production use. --- # TiDB Dashboard Top SQL Page diff --git a/data-type-date-and-time.md b/data-type-date-and-time.md index f754a1bb42f3a..b4b45f96c2da9 100644 --- a/data-type-date-and-time.md +++ b/data-type-date-and-time.md @@ -1,7 +1,6 @@ --- title: Date and Time Types summary: Learn about the supported date and time types. -aliases: ['/docs/dev/data-type-date-and-time/','/docs/dev/reference/sql/data-types/date-and-time/'] --- # Date and Time Types diff --git a/data-type-default-values.md b/data-type-default-values.md index be1e547958f85..24f498902fc27 100644 --- a/data-type-default-values.md +++ b/data-type-default-values.md @@ -1,7 +1,6 @@ --- title: TiDB Data Type summary: Learn about default values for data types in TiDB. -aliases: ['/docs/dev/data-type-default-values/','/docs/dev/reference/sql/data-types/default-values/'] --- # Default Values diff --git a/data-type-json.md b/data-type-json.md index f602582a98ba8..1c902d389765f 100644 --- a/data-type-json.md +++ b/data-type-json.md @@ -1,7 +1,6 @@ --- title: TiDB Data Type summary: Learn about the JSON data type in TiDB. -aliases: ['/docs/dev/data-type-json/','/docs/dev/reference/sql/data-types/json/'] --- # JSON Type @@ -29,7 +28,7 @@ For more information, see [JSON Functions](/functions-and-operators/json-functio ## Restrictions - Currently, TiDB only supports pushing down limited `JSON` functions to TiFlash. For more information, see [Push-down expressions](/tiflash/tiflash-supported-pushdown-calculations.md#push-down-expressions). -- TiDB Backup & Restore (BR) versions earlier than v6.3.0 do not support recovering data containing JSON columns. No version of BR supports recovering data containing JSON columns to TiDB clusters earlier than v6.3.0. +- TiDB Backup & Restore (BR) changes how JSON column data is encoded in v6.3.0. Therefore, it is not recommended to use BR to restore data containing JSON columns to a TiDB cluster earlier than v6.3.0. - Do not use any replication tool to replicate data containing non-standard `JSON` data types, such as `DATE`, `DATETIME`, and `TIME`. ## MySQL compatibility diff --git a/data-type-numeric.md b/data-type-numeric.md index 4406cc06f707a..1b7d69b19b74a 100644 --- a/data-type-numeric.md +++ b/data-type-numeric.md @@ -1,7 +1,6 @@ --- title: Numeric Types summary: Learn about numeric data types supported in TiDB. -aliases: ['/docs/dev/data-type-numeric/','/docs/dev/reference/sql/data-types/numeric/'] --- # Numeric Types diff --git a/data-type-overview.md b/data-type-overview.md index d0fe1b8c59586..07260fd94a579 100644 --- a/data-type-overview.md +++ b/data-type-overview.md @@ -1,7 +1,6 @@ --- title: Data Types summary: Learn about the data types supported in TiDB. -aliases: ['/docs/dev/data-type-overview/','/docs/dev/reference/sql/data-types/overview/'] --- # Data Types diff --git a/data-type-string.md b/data-type-string.md index 65dfae1666587..ce3b54dceba79 100644 --- a/data-type-string.md +++ b/data-type-string.md @@ -1,7 +1,6 @@ --- title: String types summary: Learn about the string types supported in TiDB. -aliases: ['/docs/dev/data-type-string/','/docs/dev/reference/sql/data-types/string/'] --- # String Types @@ -56,12 +55,12 @@ TINYTEXT [CHARACTER SET charset_name] [COLLATE collation_name] -The `MEDIUMTEXT` type is similar to the [`TEXT` type](#text-type). The difference is that the maximum column length of `MEDIUMTEXT` is 16,777,215. But due to the limitation of [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. +The `MEDIUMTEXT` type is similar to the [`TEXT` type](#text-type). The difference is that the maximum column length of `MEDIUMTEXT` is 16,777,215. But due to the limitation of [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v4010-and-v500), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. -The `MEDIUMTEXT` type is similar to the [`TEXT` type](#text-type). The difference is that the maximum column length of `MEDIUMTEXT` is 16,777,215. But due to the limitation of [`txn-entry-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-entry-size-limit-new-in-v50), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. +The `MEDIUMTEXT` type is similar to the [`TEXT` type](#text-type). The difference is that the maximum column length of `MEDIUMTEXT` is 16,777,215. But due to the limitation of [`txn-entry-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-entry-size-limit-new-in-v4010-and-v500), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. @@ -73,12 +72,12 @@ MEDIUMTEXT [CHARACTER SET charset_name] [COLLATE collation_name] -The `LONGTEXT` type is similar to the [`TEXT` type](#text-type). The difference is that the maximum column length of `LONGTEXT` is 4,294,967,295. But due to the limitation of [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. +The `LONGTEXT` type is similar to the [`TEXT` type](#text-type). The difference is that the maximum column length of `LONGTEXT` is 4,294,967,295. But due to the limitation of [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v4010-and-v500), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. -The `LONGTEXT` type is similar to the [`TEXT` type](#text-type). The difference is that the maximum column length of `LONGTEXT` is 4,294,967,295. But due to the limitation of [`txn-entry-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-entry-size-limit-new-in-v50), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. +The `LONGTEXT` type is similar to the [`TEXT` type](#text-type). The difference is that the maximum column length of `LONGTEXT` is 4,294,967,295. But due to the limitation of [`txn-entry-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-entry-size-limit-new-in-v4010-and-v500), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. @@ -122,12 +121,12 @@ TINYBLOB -The `MEDIUMBLOB` type is similar to the [`BLOB` type](#blob-type). The difference is that the maximum column length of `MEDIUMBLOB` is 16,777,215. But due to the limitation of [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. +The `MEDIUMBLOB` type is similar to the [`BLOB` type](#blob-type). The difference is that the maximum column length of `MEDIUMBLOB` is 16,777,215. But due to the limitation of [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v4010-and-v500), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. -The `MEDIUMBLOB` type is similar to the [`BLOB` type](#blob-type). The difference is that the maximum column length of `MEDIUMBLOB` is 16,777,215. But due to the limitation of [`txn-entry-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-entry-size-limit-new-in-v50), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. +The `MEDIUMBLOB` type is similar to the [`BLOB` type](#blob-type). The difference is that the maximum column length of `MEDIUMBLOB` is 16,777,215. But due to the limitation of [`txn-entry-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-entry-size-limit-new-in-v4010-and-v500), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. @@ -139,12 +138,12 @@ MEDIUMBLOB -The `LONGBLOB` type is similar to the [`BLOB` type](#blob-type). The difference is that the maximum column length of `LONGBLOB` is 4,294,967,295. But due to the limitation of [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. +The `LONGBLOB` type is similar to the [`BLOB` type](#blob-type). The difference is that the maximum column length of `LONGBLOB` is 4,294,967,295. But due to the limitation of [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v4010-and-v500), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. -The `LONGBLOB` type is similar to the [`BLOB` type](#blob-type). The difference is that the maximum column length of `LONGBLOB` is 4,294,967,295. But due to the limitation of [`txn-entry-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-entry-size-limit-new-in-v50), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. +The `LONGBLOB` type is similar to the [`BLOB` type](#blob-type). The difference is that the maximum column length of `LONGBLOB` is 4,294,967,295. But due to the limitation of [`txn-entry-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-entry-size-limit-new-in-v4010-and-v500), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. diff --git a/deploy-monitoring-services.md b/deploy-monitoring-services.md index 6c91115edd658..3a36c791f21fc 100644 --- a/deploy-monitoring-services.md +++ b/deploy-monitoring-services.md @@ -1,14 +1,11 @@ --- title: Deploy Monitoring Services for the TiDB Cluster summary: Learn how to deploy monitoring services for the TiDB cluster. -aliases: ['/docs/dev/deploy-monitoring-services/','/docs/dev/how-to/monitor/monitor-a-cluster/','/docs/dev/monitor-a-tidb-cluster/'] --- # Deploy Monitoring Services for the TiDB Cluster -This document is intended for users who want to manually deploy TiDB monitoring and alert services. - -If you deploy the TiDB cluster using TiUP, the monitoring and alert services are automatically deployed, and no manual deployment is needed. +This document is intended for users who want to manually deploy TiDB monitoring and alert services. If you deploy the TiDB cluster using TiUP, the monitoring and alert services are automatically deployed, and no manual deployment is needed. [TiDB Dashboard](/dashboard/dashboard-intro.md) is built into the PD component and does not require an independent deployment. ## Deploy Prometheus and Grafana @@ -29,9 +26,9 @@ Assume that the TiDB cluster topology is as follows: ```bash # Downloads the package. -wget https://download.pingcap.org/prometheus-2.27.1.linux-amd64.tar.gz -wget https://download.pingcap.org/node_exporter-v1.3.1-linux-amd64.tar.gz -wget https://download.pingcap.org/grafana-7.5.11.linux-amd64.tar.gz +wget https://download.pingcap.com/prometheus-2.27.1.linux-amd64.tar.gz +wget https://download.pingcap.com/node_exporter-v1.3.1-linux-amd64.tar.gz +wget https://download.pingcap.com/grafana-7.5.11.linux-amd64.tar.gz ``` {{< copyable "shell-regular" >}} @@ -226,7 +223,7 @@ To import a Grafana dashboard for the PD server, the TiKV server, and the TiDB s 2. In the sidebar menu, click **Dashboards** -> **Import** to open the **Import Dashboard** window. -3. Click **Upload .json File** to upload a JSON file (Download TiDB Grafana configuration files from [pingcap/tidb](https://github.com/pingcap/tidb/tree/master/pkg/metrics/grafana), [tikv/tikv](https://github.com/tikv/tikv/tree/master/metrics/grafana), and [tikv/pd](https://github.com/tikv/pd/tree/master/metrics/grafana)). +3. Click **Upload .json File** to upload a JSON file (Download TiDB Grafana configuration files from [pingcap/tidb](https://github.com/pingcap/tidb/tree/release-7.5/pkg/metrics/grafana), [tikv/tikv](https://github.com/tikv/tikv/tree/release-7.5/metrics/grafana), and [tikv/pd](https://github.com/tikv/pd/tree/release-7.5/metrics/grafana)). > **Note:** > diff --git a/develop/dev-guide-aws-appflow-integration.md b/develop/dev-guide-aws-appflow-integration.md index 1305234a01a60..49bbec88ba58e 100644 --- a/develop/dev-guide-aws-appflow-integration.md +++ b/develop/dev-guide-aws-appflow-integration.md @@ -7,9 +7,9 @@ summary: Introduce how to integrate TiDB with Amazon AppFlow step by step. [Amazon AppFlow](https://aws.amazon.com/appflow/) is a fully managed API integration service that you use to connect your software as a service (SaaS) applications to AWS services, and securely transfer data. With Amazon AppFlow, you can import and export data from and to TiDB into many types of data providers, such as Salesforce, Amazon S3, LinkedIn, and GitHub. For more information, see [Supported source and destination applications](https://docs.aws.amazon.com/appflow/latest/userguide/app-specific.html) in AWS documentation. -This document describes how to integrate TiDB with Amazon AppFlow and takes integrating a TiDB Serverless cluster as an example. +This document describes how to integrate TiDB with Amazon AppFlow and takes integrating a {{{ .starter }}} cluster as an example. -If you do not have a TiDB cluster, you can create a [TiDB Serverless](https://tidbcloud.com/console/clusters) cluster, which is free and can be created in approximately 30 seconds. +If you do not have a TiDB cluster, you can create a [{{{ .starter }}}](https://tidbcloud.com/console/clusters) cluster, which is free and can be created in approximately 30 seconds. ## Prerequisites @@ -66,7 +66,7 @@ git clone https://github.com/pingcap-inc/tidb-appflow-integration > > - The `--guided` option uses prompts to guide you through the deployment. Your input will be stored in a configuration file, which is `samconfig.toml` by default. > - `stack_name` specifies the name of AWS Lambda that you are deploying. - > - This prompted guide uses AWS as the cloud provider of TiDB Serverless. To use Amazon S3 as the source or destination, you need to set the `region` of AWS Lambda as the same as that of Amazon S3. + > - This prompted guide uses AWS as the cloud provider of {{{ .starter }}}. To use Amazon S3 as the source or destination, you need to set the `region` of AWS Lambda as the same as that of Amazon S3. > - If you have already run `sam deploy --guided` before, you can just run `sam deploy` instead, and SAM CLI will use the configuration file `samconfig.toml` to simplify the interaction. If you see a similar output as follows, this Lambda is successfully deployed. @@ -148,7 +148,7 @@ Choose the **Source details** and **Destination details**. TiDB connector can be ``` 5. After the `sf_account` table is created, click **Connect**. A connection dialog is displayed. -6. In the **Connect to TiDB-Connector** dialog, enter the connection properties of the TiDB cluster. If you use a TiDB Serverless cluster, you need to set the **TLS** option to `Yes`, which lets the TiDB connector use the TLS connection. Then, click **Connect**. +6. In the **Connect to TiDB-Connector** dialog, enter the connection properties of the TiDB cluster. If you use a {{{ .starter }}} cluster, you need to set the **TLS** option to `Yes`, which lets the TiDB connector use the TLS connection. Then, click **Connect**. ![tidb connection message](/media/develop/aws-appflow-step-tidb-connection-message.png) @@ -244,5 +244,19 @@ test> SELECT * FROM sf_account; - If anything goes wrong, you can navigate to the [CloudWatch](https://console.aws.amazon.com/cloudwatch/home) page on the AWS Management Console to get logs. - The steps in this document are based on [Building custom connectors using the Amazon AppFlow Custom Connector SDK](https://aws.amazon.com/blogs/compute/building-custom-connectors-using-the-amazon-appflow-custom-connector-sdk/). -- [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) is **NOT** a production environment. -- To prevent excessive length, the examples in this document only show the `Insert` strategy, but `Update` and `Upsert` strategies are also tested and can be used. \ No newline at end of file +- [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) is **NOT** a production environment. +- To prevent excessive length, the examples in this document only show the `Insert` strategy, but `Update` and `Upsert` strategies are also tested and can be used. + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-bookshop-schema-design.md b/develop/dev-guide-bookshop-schema-design.md index e513cbafd7293..9a457d3b6ae96 100644 --- a/develop/dev-guide-bookshop-schema-design.md +++ b/develop/dev-guide-bookshop-schema-design.md @@ -1,5 +1,6 @@ --- title: Bookshop Example Application +summary: Bookshop is an online bookstore app for buying and rating books. You can import table structures and data via TiUP or TiDB Cloud. Method 1 uses TiUP to quickly generate and import sample data, while Method 2 imports data from Amazon S3 to TiDB Cloud. The database tables include books, authors, users, ratings, book_authors, and orders. The database initialization script `dbinit.sql` creates the table structures for the Bookshop application. --- # Bookshop Example Application @@ -88,26 +89,26 @@ You can delete the original table structure through the `--drop-tables` paramete ### Method 2: Via TiDB Cloud Import -On the cluster detail page of TiDB Cloud, click **Import Data** in the **Import** area to enter the **Data Import** page. On this page, perform the following steps to import the Bookshop sample data from AWS S3 to TiDB Cloud. +1. Open the **Import** page for your target cluster. -1. Select **SQL File** for **Data Format**. -2. Copy the following **Bucket URI** and **Role ARN** to the corresponding input boxes: + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. - **Bucket URI**: + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. - ``` - s3://developer.pingcap.com/bookshop/ - ``` + 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. - **Role ARN**: +2. Select **Import data from Cloud Storage**, and then click **Amazon S3**. - ``` - arn:aws:iam::494090988690:role/s3-tidb-cloud-developer-access - ``` +3. On the **Import Data from Amazon S3** page, configure the following source data information: -3. Click **Next** to go to the **File and filter** step to confirm the information of the files to be imported. - -4. Click **Next** again to go to the **Preview** step to confirm the preview of the data to be imported. + - **Import File Count**: for {{{ .starter }}}, select **Multiple files**. This field is not available in TiDB Cloud Dedicated. + - **Included Schema Files**: select **Yes**. + - **Data Format**: select **SQL**. + - **Folder URI**: enter `s3://developer.pingcap.com/bookshop/`. + - **Bucket Access**: select **AWS Role ARN**. + - **Role ARN**: enter `arn:aws:iam::494090988690:role/s3-tidb-cloud-developer-access`. In this example, the following data is generated in advance: @@ -117,7 +118,7 @@ On the cluster detail page of TiDB Cloud, click **Import Data** in the **Import* - 1,000,000 rows of rating records - 1,000,000 rows of order records -5. Click **Start Import** to start the import process and wait for TiDB Cloud to complete the import. +4. Click **Connect** > **Start Import** to start the import process and wait for TiDB Cloud to complete the import. For more information about how to import or migrate data to TiDB Cloud, see [TiDB Cloud Migration Overview](https://docs.pingcap.com/tidbcloud/tidb-cloud-migration-overview). @@ -289,3 +290,7 @@ CREATE TABLE `bookshop`.`orders` ( KEY `orders_book_id_idx` (`book_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin ``` + +## Need help? + +Ask questions on [TiDB Community](https://ask.pingcap.com/). diff --git a/develop/dev-guide-build-cluster-in-cloud.md b/develop/dev-guide-build-cluster-in-cloud.md index fb3e9733a32a3..cec6c4aa480ff 100644 --- a/develop/dev-guide-build-cluster-in-cloud.md +++ b/develop/dev-guide-build-cluster-in-cloud.md @@ -1,15 +1,15 @@ --- -title: Build a TiDB Serverless Cluster -summary: Learn how to build a TiDB Serverless cluster in TiDB Cloud and connect to it. +title: Build a {{{ .starter }}} Cluster +summary: Learn how to build a {{{ .starter }}} cluster in TiDB Cloud and connect to it. --- -# Build a TiDB Serverless Cluster +# Build a {{{ .starter }}} Cluster -This document walks you through the quickest way to get started with TiDB. You will use [TiDB Cloud](https://en.pingcap.com/tidb-cloud) to create a TiDB Serverless cluster, connect to it, and run a sample application on it. +This document walks you through the quickest way to get started with TiDB. You will use [TiDB Cloud](https://www.pingcap.com/tidb-cloud) to create a {{{ .starter }}} cluster, connect to it, and run a sample application on it. If you need to run TiDB on your local machine, see [Starting TiDB Locally](/quick-start-with-tidb.md). @@ -21,7 +21,7 @@ This document walks you through the quickest way to get started with TiDB Cloud. -## Step 1. Create a TiDB Serverless cluster +## Step 1. Create a {{{ .starter }}} cluster {#step-1-create-a-tidb-cloud-cluster} 1. If you do not have a TiDB Cloud account, click [here](https://tidbcloud.com/free-trial) to sign up for an account. @@ -29,9 +29,9 @@ This document walks you through the quickest way to get started with TiDB Cloud. 3. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click **Create Cluster**. -4. On the **Create Cluster** page, **Serverless** is selected by default. Update the default cluster name if necessary, and then select the region where you want to create your cluster. +4. On the **Create Cluster** page, **Starter** is selected by default. Update the default cluster name if necessary, and then select the region where you want to create your cluster. -5. Click **Create** to create a TiDB Serverless cluster. +5. Click **Create** to create a {{{ .starter }}} cluster. Your TiDB Cloud cluster will be created in approximately 30 seconds. @@ -39,13 +39,13 @@ This document walks you through the quickest way to get started with TiDB Cloud. 7. In the dialog, select your preferred connection method and operating system to get the corresponding connection string. This document uses MySQL client as an example. -8. Click **Create password** to generate a random password. The generated password will not show again, so save your password in a secure location. If you do not set a root password, you cannot connect to the cluster. +8. Click **Generate Password** to generate a random password. The generated password will not show again, so save your password in a secure location. If you do not set a root password, you cannot connect to the cluster. > **Note:** > -> For [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters, when you connect to your cluster, you must include the prefix for your cluster in the user name and wrap the name with quotation marks. For more information, see [User name prefix](https://docs.pingcap.com/tidbcloud/select-cluster-tier#user-name-prefix). +> For [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) clusters, when you connect to your cluster, you must include the prefix for your cluster in the user name and wrap the name with quotation marks. For more information, see [User name prefix](https://docs.pingcap.com/tidbcloud/select-cluster-tier#user-name-prefix). @@ -53,7 +53,7 @@ This document walks you through the quickest way to get started with TiDB Cloud. > **Note:** > -> For [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters, when you connect to your cluster, you must include the prefix for your cluster in the user name and wrap the name with quotation marks. For more information, see [User name prefix](/tidb-cloud/select-cluster-tier.md#user-name-prefix). +> For [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) clusters, when you connect to your cluster, you must include the prefix for your cluster in the user name and wrap the name with quotation marks. For more information, see [User name prefix](/tidb-cloud/select-cluster-tier.md#user-name-prefix). @@ -130,7 +130,7 @@ mysql Ver 15.1 Distrib 5.5.68-MariaDB, for Linux (x86_64) using readline 5.1 -2. Run the connection string obtained in [Step 1](#step-1-create-a-tidb-serverless-cluster). +2. Run the connection string obtained in [Step 1](#step-1-create-a-tidb-cloud-cluster). {{< copyable "shell-regular" >}} @@ -142,8 +142,8 @@ mysql Ver 15.1 Distrib 5.5.68-MariaDB, for Linux (x86_64) using readline 5.1 > **Note:** > -> - When you connect to a TiDB Serverless cluster, you must [use the TLS connection](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters). -> - If you encounter problems when connecting to a TiDB Serverless cluster, you can read [Secure Connections to TiDB Serverless Clusters](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters) for more information. +> - When you connect to a {{{ .starter }}} cluster, you must [use the TLS connection](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters). +> - If you encounter problems when connecting to a {{{ .starter }}} cluster, you can read [Secure Connections to {{{ .starter }}} Clusters](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters) for more information. @@ -151,8 +151,8 @@ mysql Ver 15.1 Distrib 5.5.68-MariaDB, for Linux (x86_64) using readline 5.1 > **Note:** > -> - When you connect to a TiDB Serverless cluster, you must [use the TLS connection](/tidb-cloud/secure-connections-to-serverless-clusters.md). -> - If you encounter problems when connecting to a TiDB Serverless cluster, you can read [Secure Connections to TiDB Serverless Clusters](/tidb-cloud/secure-connections-to-serverless-clusters.md) for more information. +> - When you connect to a {{{ .starter }}} cluster, you must [use the TLS connection](/tidb-cloud/secure-connections-to-serverless-clusters.md). +> - If you encounter problems when connecting to a {{{ .starter }}} cluster, you can read [Secure Connections to {{{ .starter }}} Clusters](/tidb-cloud/secure-connections-to-serverless-clusters.md) for more information. @@ -177,3 +177,17 @@ Expected output: ``` If your actual output is similar to the expected output, congratulations, you have successfully execute a SQL statement on TiDB Cloud. + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-choose-driver-or-orm.md b/develop/dev-guide-choose-driver-or-orm.md index b8b4d585a4cea..316ae9a9036cd 100644 --- a/develop/dev-guide-choose-driver-or-orm.md +++ b/develop/dev-guide-choose-driver-or-orm.md @@ -27,11 +27,15 @@ This section describes how to use drivers and ORM frameworks in Java. Support level: **Full** -You can follow the [MySQL documentation](https://dev.mysql.com/doc/connector-j/en/) to download and configure a Java JDBC driver. It is recommended to use MySQL Connector/J 8.0.33 or later with TiDB v6.3.0 and newer. +You can follow the [MySQL documentation](https://dev.mysql.com/doc/connector-j/en/) to download and configure a Java JDBC driver. It is recommended to use the latest GA version of MySQL Connector/J with TiDB v6.3.0 or later. -> **Tip:** +> **Warning:** > -> There is a [bug](https://bugs.mysql.com/bug.php?id=106252) in the Connector/J 8.0 versions before 8.0.32, which might cause threads to hang when using TiDB versions earlier than v6.3.0. To avoid this issue, it is recommended that you use either MySQL Connector/J 8.0.32 or a later version, or the TiDB JDBC (see the *TiDB-JDBC* tab). +> There is a [bug](https://bugs.mysql.com/bug.php?id=106252) in the MySQL Connector/J 8.0 versions before 8.0.31 (see [MySQL JDBC bugs](/develop/dev-guide-third-party-tools-compatibility.md#mysql-jdbc-bugs) for details), which might cause threads to hang when using TiDB versions earlier than v6.3.0. To avoid this issue, do **NOT** use MySQL Connector/J 8.0.31 or an earlier version. + +> **Note:** +> +> When you are using MySQL Connector/J 8.0 with TiDB v7.5.2 or earlier versions, it is recommended to set the TiDB configuration item [`server-version`](https://docs.pingcap.com/tidb/v7.5/tidb-configuration-file#server-version) to `"5.7.25-TiDB-v7.5.x"`. MySQL Connector/J attempts to access the [`information_schema.KEYWORDS`](/information-schema/information-schema-keywords.md) table if the TiDB server reports a version of MySQL 8.0.11 or later. However, this table is introduced starting from v7.5.3 and does not exist in earlier versions. For an example of how to build a complete application, see [Build a simple CRUD app with TiDB and JDBC](/develop/dev-guide-sample-application-java-jdbc.md). @@ -87,15 +91,15 @@ implementation group: 'org.bouncycastle', name: 'bcpkix-jdk15on', version: '1.67 ### Java ORM frameworks + +
+ > **Note:** > > - Currently, Hibernate does [not support nested transactions](https://stackoverflow.com/questions/37927208/nested-transaction-in-spring-app-with-jpa-postgres). > > - Since v6.2.0, TiDB supports [savepoint](/sql-statements/sql-statement-savepoint.md). To use the `Propagation.NESTED` transaction propagation option in `@Transactional`, that is, to set `@Transactional(propagation = Propagation.NESTED)`, make sure that your TiDB is v6.2.0 or later. - -
- Support level: **Full** To avoid manually managing complex relationships between different dependencies of an application, you can use [Gradle](https://gradle.org/install) or [Maven](https://maven.apache.org/install.html) to get all dependencies of your application, including those indirect ones. Note that only Hibernate `6.0.0.Beta2` or above supports the TiDB dialect. @@ -106,21 +110,21 @@ If you are using **Maven**, add the following to your ` org.hibernate.orm hibernate-core - 6.0.0.CR2 + 6.2.3.Final mysql mysql-connector-java - 5.1.49 + 8.0.33 ``` If you are using **Gradle**, add the following to your `dependencies`: ```gradle -implementation 'org.hibernate:hibernate-core:6.0.0.CR2' -implementation 'mysql:mysql-connector-java:5.1.49' +implementation 'org.hibernate:hibernate-core:6.2.3.Final' +implementation 'mysql:mysql-connector-java:8.0.33' ``` - For an example of using Hibernate to build a TiDB application by native Java, see [Build a simple CRUD app with TiDB and Hibernate](/develop/dev-guide-sample-application-java-hibernate.md). @@ -146,21 +150,21 @@ If you are using Maven, add the following to your ` org.mybatis mybatis - 3.5.9 + 3.5.13 mysql mysql-connector-java - 5.1.49 + 8.0.33 ``` If you are using Gradle, add the following to your `dependencies`: ```gradle -implementation 'org.mybatis:mybatis:3.5.9' -implementation 'mysql:mysql-connector-java:5.1.49' +implementation 'org.mybatis:mybatis:3.5.13' +implementation 'mysql:mysql-connector-java:8.0.33' ``` For an example of using MyBatis to build a TiDB application, see [Build a simple CRUD app with TiDB and MyBatis](/develop/dev-guide-sample-application-java-mybatis.md). @@ -307,3 +311,17 @@ For an example of using peewee to build a TiDB application, see [Connect to TiDB After you have determined the driver or ORM, you can [connect to your TiDB cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-connect-to-tidb.md b/develop/dev-guide-connect-to-tidb.md index f9ed05e7f5e1e..6753a6234cc49 100644 --- a/develop/dev-guide-connect-to-tidb.md +++ b/develop/dev-guide-connect-to-tidb.md @@ -133,3 +133,7 @@ For more information about TiDB SQL users, see [TiDB User Account Management](/u For more information about TiDB SQL users, see [TiDB User Account Management](https://docs.pingcap.com/tidb/stable/user-account-management). + +## Need help? + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). diff --git a/develop/dev-guide-connection-parameters.md b/develop/dev-guide-connection-parameters.md index f3c7b8fc41aca..46ebc92f08c86 100644 --- a/develop/dev-guide-connection-parameters.md +++ b/develop/dev-guide-connection-parameters.md @@ -1,5 +1,6 @@ --- title: Connection Pools and Connection Parameters +summary: This document explains how to configure connection pools and parameters for TiDB. It covers connection pool size, probe configuration, and formulas for optimal throughput. It also discusses JDBC API usage and MySQL Connector/J parameter configurations for performance optimization. --- # Connection Pools and Connection Parameters @@ -8,7 +9,7 @@ This document describes how to configure connection pools and connection paramet -If you are interested in more tips about Java application development, see [Best Practices for Developing Java Applications with TiDB](/best-practices/java-app-best-practices.md#connection-pool) +If you are interested in more tips about Java application development, see [Best Practices for Developing Java Applications with TiDB](/develop/java-app-best-practices.md#connection-pool) @@ -35,7 +36,12 @@ The application needs to return the connection after finishing using it. It is r ### Probe configuration -The connection pool maintains persistent connections to TiDB. TiDB does not proactively close client connections by default (unless an error is reported), but generally, there are also network proxies such as [LVS](https://en.wikipedia.org/wiki/Linux_Virtual_Server) or [HAProxy](https://en.wikipedia.org/wiki/HAProxy) between the client and TiDB. Usually, these proxies proactively clean up connections that are idle for a certain period. In addition to paying attention to the idle configuration of the proxies, the connection pool also needs to keep alive or probe connections. +The connection pool maintains persistent connections from clients to TiDB as follows: + +- Before v5.4, TiDB does not proactively close client connections by default (unless an error is reported). +- Starting from v5.4, TiDB automatically closes client connections after `28800` seconds (this is, `8` hours) of inactivity by default. You can control this timeout setting using the TiDB and MySQL compatible `wait_timeout` variable. For more information, see [JDBC Query Timeout](/develop/dev-guide-timeouts-in-tidb.md#jdbc-query-timeout). + +Moreover, there might be network proxies such as [LVS](https://en.wikipedia.org/wiki/Linux_Virtual_Server) or [HAProxy](https://en.wikipedia.org/wiki/HAProxy) between clients and TiDB. These proxies typically proactively clean up connections after a specific idle period (determined by the proxy's idle configuration). In addition to monitoring the proxy's idle configuration, connection pools also need to maintain or probe connections for keep-alive. If you often see the following error in your Java application: @@ -151,7 +157,7 @@ TiDB supports both methods, but it is preferred that you use the first method, b ### MySQL JDBC parameters -JDBC usually provides implementation-related configurations in the form of JDBC URL parameters. This section introduces [MySQL Connector/J's parameter configurations](https://dev.mysql.com/doc/connector-j/en/connector-j-reference-configuration-properties.html) (If you use MariaDB, see [MariaDB's parameter configurations](https://mariadb.com/kb/en/library/about-mariadb-connector-j/#optional-url-parameters)). Because this document cannot cover all configuration items, it mainly focuses on several parameters that might affect performance. +JDBC usually provides implementation-related configurations in the form of JDBC URL parameters. This section introduces [MySQL Connector/J's parameter configurations](https://dev.mysql.com/doc/connector-j/en/connector-j-reference-configuration-properties.html) (If you use MariaDB, see [MariaDB's parameter configurations](https://mariadb.com/docs/connectors/mariadb-connector-j/about-mariadb-connector-j#optional-url-parameters)). Because this document cannot cover all configuration items, it mainly focuses on several parameters that might affect performance. #### Prepare-related parameters @@ -268,3 +274,17 @@ After it is configured, you can check the monitoring to see a decreased number o TiDB provides two MySQL-compatible parameters to control the timeout: [`wait_timeout`](/system-variables.md#wait_timeout) and [`max_execution_time`](/system-variables.md#max_execution_time). These two parameters respectively control the connection idle timeout with the Java application and the timeout of the SQL execution in the connection; that is to say, these parameters control the longest idle time and the longest busy time for the connection between TiDB and the Java application. Since TiDB v5.4, the default value of `wait_timeout` is `28800` seconds, which is 8 hours. For TiDB versions earlier than v5.4, the default value is `0`, which means the timeout is unlimited. The default value of `max_execution_time` is `0`, which means the maximum execution time of a SQL statement is unlimited. However, in an actual production environment, idle connections and SQL statements with excessively long execution time negatively affect databases and applications. To avoid idle connections and SQL statements that are executed for too long, you can configure these two parameters in your application's connection string. For example, set `sessionVariables=wait_timeout=3600` (1 hour) and `sessionVariables=max_execution_time=300000` (5 minutes). + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-create-database.md b/develop/dev-guide-create-database.md index c9ef93d081af3..1f0617d832738 100644 --- a/develop/dev-guide-create-database.md +++ b/develop/dev-guide-create-database.md @@ -11,7 +11,7 @@ This document describes how to create a database using SQL and various programmi Before creating a database, do the following: -- [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). +- [Build a {{{ .starter }}} Cluster](/develop/dev-guide-build-cluster-in-cloud.md). - Read [Schema Design Overview](/develop/dev-guide-schema-design-overview.md). ## What is database @@ -80,3 +80,17 @@ The following is an example output: ## Next step After creating a database, you can add **tables** to it. For more information, see [Create a Table](/develop/dev-guide-create-table.md). + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-create-secondary-indexes.md b/develop/dev-guide-create-secondary-indexes.md index 0e57fd3c6f4a9..777bcc25ab34a 100644 --- a/develop/dev-guide-create-secondary-indexes.md +++ b/develop/dev-guide-create-secondary-indexes.md @@ -11,7 +11,7 @@ This document describes how to create a secondary index using SQL and various pr Before creating a secondary index, do the following: -- [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). +- [Build a {{{ .starter }}} Cluster](/develop/dev-guide-build-cluster-in-cloud.md). - Read [Schema Design Overview](/develop/dev-guide-schema-design-overview.md). - [Create a Database](/develop/dev-guide-create-database.md). - [Create a Table](/develop/dev-guide-create-table.md). @@ -183,3 +183,17 @@ The following is an example output: ## Next step After creating a database and adding tables and secondary indexes to it, you can start adding the data [write](/develop/dev-guide-insert-data.md) and [read](/develop/dev-guide-get-data-from-single-table.md) features to your application. + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-create-table.md b/develop/dev-guide-create-table.md index 058828e5a82f5..ef6aef0db98df 100644 --- a/develop/dev-guide-create-table.md +++ b/develop/dev-guide-create-table.md @@ -11,7 +11,7 @@ This document introduces how to create tables using the SQL statement and the re Before reading this document, make sure that the following tasks are completed: -- [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). +- [Build a {{{ .starter }}} Cluster](/develop/dev-guide-build-cluster-in-cloud.md). - Read [Schema Design Overview](/develop/dev-guide-schema-design-overview.md). - [Create a Database](/develop/dev-guide-create-database.md). @@ -106,7 +106,7 @@ A [primary key](/constraints.md#primary-key) is a column or a set of columns in > > - In **InnoDB**: A **primary key** is unique, not null, and **index clustered**. > -> - In TiDB: A **primary key** is unique and is not null. But the primary key is not guaranteed to be a **clustered index**. Instead, another set of keywords `CLUSTERED` / `NONCLUSTERED` additionally controls whether the **primary key** is a **clustered index**. If the keyword is not specified, it is controlled by the system variable `@@global.tidb_enable_clustered_index`, as described in [clustered indexes](https://docs.pingcap.com/zh/tidb/stable/clustered-indexes). +> - In TiDB: A **primary key** is unique and is not null. But the primary key is not guaranteed to be a **clustered index**. Instead, another set of keywords `CLUSTERED` / `NONCLUSTERED` additionally controls whether the **primary key** is a **clustered index**. If the keyword is not specified, it is controlled by the system variable `@@global.tidb_enable_clustered_index`, as described in [clustered indexes](https://docs.pingcap.com/tidb/stable/clustered-indexes). The **primary key** is defined in the `CREATE TABLE` statement. The [primary key constraint](/constraints.md#primary-key) requires that all constrained columns contain only non-NULL values. @@ -135,7 +135,7 @@ CREATE TABLE `bookshop`.`users` ( TiDB supports the [clustered index](/clustered-indexes.md) feature since v5.0. This feature controls how data is stored in tables containing primary keys. It provides TiDB the ability to organize tables in a way that can improve the performance of certain queries. -The term clustered in this context refers to the organization of how data is stored and not a group of database servers working together. Some database management systems refer to clustered indexes as index-organized tables (IOT). +The term clustered in this context refers to the organization of how data is stored and not a group of database servers working together. Some database management systems refer to clustered index tables as index-organized tables (IOT). Currently, tables **_containing primary_** keys in TiDB are divided into the following two categories: @@ -290,7 +290,7 @@ ALTER TABLE `bookshop`.`ratings` SET TIFLASH REPLICA 1; > **Note:** > -> If your cluster does not contain **TiFlash** nodes, this SQL statement will report an error: `1105 - the tiflash replica count: 1 should be less than the total tiflash server count: 0`. You can use [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster) to create a TiDB Serverless cluster that includes **TiFlash**. +> If your cluster does not contain **TiFlash** nodes, this SQL statement will report an error: `1105 - the tiflash replica count: 1 should be less than the total tiflash server count: 0`. You can use [Build a {{{ .starter }}} Cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-cloud-cluster) to create a {{{ .starter }}} cluster that includes **TiFlash**. Then you can go on to perform the following query: @@ -407,3 +407,17 @@ This section provides guidelines you need to follow when creating a table. ## One more step Note that all the tables that have been created in this document do not contain secondary indexes. For a guide to add secondary indexes, refer to [Creating Secondary Indexes](/develop/dev-guide-create-secondary-indexes.md). + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-delete-data.md b/develop/dev-guide-delete-data.md index 96278a73313a5..421d28a2fc324 100644 --- a/develop/dev-guide-delete-data.md +++ b/develop/dev-guide-delete-data.md @@ -11,7 +11,7 @@ This document describes how to use the [DELETE](/sql-statements/sql-statement-de Before reading this document, you need to prepare the following: -- [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md) +- [Build a {{{ .starter }}} Cluster](/develop/dev-guide-build-cluster-in-cloud.md) - Read [Schema Design Overview](/develop/dev-guide-schema-design-overview.md), [Create a Database](/develop/dev-guide-create-database.md), [Create a Table](/develop/dev-guide-create-table.md), and [Create Secondary Indexes](/develop/dev-guide-create-secondary-indexes.md) - [Insert Data](/develop/dev-guide-insert-data.md) @@ -411,3 +411,17 @@ In the same scenario as the [Bulk-delete example](#bulk-delete-example), the fol ```sql BATCH ON `rated_at` LIMIT 1000 DELETE FROM `ratings` WHERE `rated_at` >= "2022-04-15 00:00:00" AND `rated_at` <= "2022-04-15 00:15:00"; ``` + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-get-data-from-single-table.md b/develop/dev-guide-get-data-from-single-table.md index 2f3f6bf6f3e57..77c6a03b10e86 100644 --- a/develop/dev-guide-get-data-from-single-table.md +++ b/develop/dev-guide-get-data-from-single-table.md @@ -395,4 +395,18 @@ The result is as follows: 71 rows in set (0.00 sec) ``` -In addition to the `COUNT` function, TiDB also supports other aggregate functions. For more information, see [Aggregate (GROUP BY) Functions](/functions-and-operators/aggregate-group-by-functions.md). \ No newline at end of file +In addition to the `COUNT` function, TiDB also supports other aggregate functions. For more information, see [Aggregate (GROUP BY) Functions](/functions-and-operators/aggregate-group-by-functions.md). + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-gui-datagrip.md b/develop/dev-guide-gui-datagrip.md index 3ed6652fbfd2d..62bcb471521db 100644 --- a/develop/dev-guide-gui-datagrip.md +++ b/develop/dev-guide-gui-datagrip.md @@ -9,7 +9,7 @@ TiDB is a MySQL-compatible database, and [JetBrains DataGrip](https://www.jetbra > **Note:** > -> This tutorial is compatible with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial is compatible with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed. You can use DataGrip in two ways: @@ -29,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -37,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -47,7 +47,7 @@ To complete this tutorial, you need: Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -55,15 +55,16 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` - - **Connect With** is set to `JDBC` + - **Connection Type** is set to `Public` + - **Branch** is set to `main` + - **Connect With** is set to `DataGrip` - **Operating System** matches your environment. -4. Click **Create password** to create a random password. +4. Click **Generate Password** to create a random password. > **Tip:** > - > If you have created a password before, you can either use the original password or click **Reset password** to generate a new one. + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. 5. Launch DataGrip and create a project to manage your connections. @@ -73,28 +74,30 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele ![Select a data source in DataGrip](/media/develop/datagrip-data-source-select.jpg) -7. Copy the JDBC string from the TiDB Cloud connection dialog and replace `` with your actual password. Then, paste it into the **URL** field, and the remaining parameters will be auto-populated. An example result is as follows: +7. Copy the connection string from the TiDB Cloud connection dialog. Then, paste it into the **URL** field, and the remaining parameters will be auto-populated. An example result is as follows: - ![Configure the URL field for TiDB Serverless](/media/develop/datagrip-url-paste.jpg) + ![Configure the URL field for {{{ .starter }}}](/media/develop/datagrip-url-paste.jpg) If a **Download missing driver files** warning displays, click **Download** to acquire the driver files. -8. Click **Test Connection** to validate the connection to the TiDB Serverless cluster. +8. Click **Test Connection** to validate the connection to the {{{ .starter }}} cluster. - ![Test the connection to a TiDB Serverless clustser](/media/develop/datagrip-test-connection.jpg) + ![Test the connection to a {{{ .starter }}} clustser](/media/develop/datagrip-test-connection.jpg) 9. Click **OK** to save the connection configuration.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Launch DataGrip and create a project to manage your connections. @@ -104,9 +107,9 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele ![Select a data source in DataGrip](/media/develop/datagrip-data-source-select.jpg) -6. Copy and paste the appropriate connection string into the **Data Source and Drivers** window in DataGrip. The mappings between DataGrip fields and TiDB Dedicated connection string are as follows: +6. Copy and paste the appropriate connection string into the **Data Source and Drivers** window in DataGrip. The mappings between DataGrip fields and TiDB Cloud Dedicated connection string are as follows: - | DataGrip field | TiDB Dedicated connection string | + | DataGrip field | TiDB Cloud Dedicated connection string | | -------------- | ------------------------------- | | Host | `{host}` | | Port | `{port}` | @@ -115,26 +118,26 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele An example is as follows: - ![Configure the connection parameters for TiDB Dedicated](/media/develop/datagrip-dedicated-connect.jpg) + ![Configure the connection parameters for TiDB Cloud Dedicated](/media/develop/datagrip-dedicated-connect.jpg) 7. Click the **SSH/SSL** tab, select the **Use SSL** checkbox, and input the CA certificate path into the **CA file** field. - ![Configure the CA for TiDB Dedicated](/media/develop/datagrip-dedicated-ssl.jpg) + ![Configure the CA for TiDB Cloud Dedicated](/media/develop/datagrip-dedicated-ssl.jpg) If a **Download missing driver files** warning displays, click **Download** to acquire the driver files. 8. Click the **Advanced** tab, scroll to find the **enabledTLSProtocols** parameter, and set its value to `TLSv1.2,TLSv1.3`. - ![Configure the TLS for TiDB Dedicated](/media/develop/datagrip-dedicated-advanced.jpg) + ![Configure the TLS for TiDB Cloud Dedicated](/media/develop/datagrip-dedicated-advanced.jpg) -9. Click **Test Connection** to validate the connection to the TiDB Dedicated cluster. +9. Click **Test Connection** to validate the connection to the TiDB Cloud Dedicated cluster. - ![Test the connection to a TiDB Dedicated cluster](/media/develop/datagrip-dedicated-test-connection.jpg) + ![Test the connection to a TiDB Cloud Dedicated cluster](/media/develop/datagrip-dedicated-test-connection.jpg) 10. Click **OK** to save the connection configuration.
-
+
1. Launch DataGrip and create a project to manage your connections. @@ -146,20 +149,20 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Configure the following connection parameters: - - **Host**: The IP address or domain name of your TiDB Self-Hosted cluster. - - **Port**: The port number of your TiDB Self-Hosted cluster. - - **User**: The username to use to connect to your TiDB Self-Hosted cluster. + - **Host**: The IP address or domain name of your TiDB Self-Managed cluster. + - **Port**: The port number of your TiDB Self-Managed cluster. + - **User**: The username to use to connect to your TiDB Self-Managed cluster. - **Password**: The password of the username. An example is as follows: - ![Configure the connection parameters for TiDB Self-Hosted](/media/develop/datagrip-self-hosted-connect.jpg) + ![Configure the connection parameters for TiDB Self-Managed](/media/develop/datagrip-self-hosted-connect.jpg) If a **Download missing driver files** warning displays, click **Download** to acquire the driver files. -4. Click **Test Connection** to validate the connection to the TiDB Self-Hosted cluster. +4. Click **Test Connection** to validate the connection to the TiDB Self-Managed cluster. - ![Test the connection to a TiDB Self-Hosted cluster](/media/develop/datagrip-self-hosted-test-connection.jpg) + ![Test the connection to a TiDB Self-Managed cluster](/media/develop/datagrip-self-hosted-test-connection.jpg) 5. Click **OK** to save the connection configuration. @@ -174,4 +177,14 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-gui-dbeaver.md b/develop/dev-guide-gui-dbeaver.md index 026e2fc0fb5c9..6b20581626b5d 100644 --- a/develop/dev-guide-gui-dbeaver.md +++ b/develop/dev-guide-gui-dbeaver.md @@ -11,7 +11,7 @@ In this tutorial, you can learn how to connect to your TiDB cluster using DBeave > **Note:** > -> This tutorial is compatible with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial is compatible with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -24,7 +24,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -32,7 +32,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -42,7 +42,7 @@ To complete this tutorial, you need: Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -50,27 +50,28 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` - - **Connect With** is set to `JDBC` + - **Connection Type** is set to `Public` + - **Branch** is set to `main` + - **Connect With** is set to `DBeaver` - **Operating System** matches your environment. -4. Click **Create password** to create a random password. +4. Click **Generate Password** to create a random password. > **Tip:** > - > If you have created a password before, you can either use the original password or click **Reset password** to generate a new one. + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. 5. Launch DBeaver and click **New Database Connection** in the upper-left corner. In the **Connect to a database** dialog, select **TiDB** from the list, and then click **Next**. ![Select TiDB as the database in DBeaver](/media/develop/dbeaver-select-database.jpg) -6. Copy the JDBC string from the TiDB Cloud connection dialog. In DBeaver, select **URL** for **Connect by** and paste the JDBC string into the **URL** field. You don't need to replace the `` placeholder in the string with your actual password, because DBeaver reads username and password from the **Authentication (Database Native)** section. +6. Copy the connection string from the TiDB Cloud connection dialog. In DBeaver, select **URL** for **Connect by** and paste the connection string into the **URL** field. 7. In the **Authentication (Database Native)** section, enter your **Username** and **Password**. An example is as follows: - ![Configure connection settings for TiDB Serverless](/media/develop/dbeaver-connection-settings-serverless.jpg) + ![Configure connection settings for {{{ .starter }}}](/media/develop/dbeaver-connection-settings-serverless.jpg) -8. Click **Test Connection** to validate the connection to the TiDB Serverless cluster. +8. Click **Test Connection** to validate the connection to the {{{ .starter }}} cluster. If the **Download driver files** dialog is displayed, click **Download** to get the driver files. @@ -83,23 +84,25 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 9. Click **Finish** to save the connection configuration.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere**, and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Launch DBeaver and click **New Database Connection** in the upper-left corner. In the **Connect to a database** dialog, select **TiDB** from the list, and then click **Next**. ![Select TiDB as the database in DBeaver](/media/develop/dbeaver-select-database.jpg) -5. Copy and paste the appropriate connection string into the DBeaver connection panel. The mappings between DBeaver fields and TiDB Dedicated connection string are as follows: +5. Copy and paste the appropriate connection string into the DBeaver connection panel. The mappings between DBeaver fields and TiDB Cloud Dedicated connection string are as follows: - | DBeaver field | TiDB Dedicated connection string | + | DBeaver field | TiDB Cloud Dedicated connection string | |---------------| ------------------------------- | | Server Host | `{host}` | | Port | `{port}` | @@ -108,9 +111,9 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele An example is as follows: - ![Configure connection settings for TiDB Dedicated](/media/develop/dbeaver-connection-settings-dedicated.jpg) + ![Configure connection settings for TiDB Cloud Dedicated](/media/develop/dbeaver-connection-settings-dedicated.jpg) -6. Click **Test Connection** to validate the connection to the TiDB Dedicated cluster. +6. Click **Test Connection** to validate the connection to the TiDB Cloud Dedicated cluster. If the **Download driver files** dialog is displayed, click **Download** to get the driver files. @@ -123,7 +126,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 7. Click **Finish** to save the connection configuration.
-
+
1. Launch DBeaver and click **New Database Connection** in the upper-left corner. In the **Connect to a database** dialog, select **TiDB** from the list, and then click **Next**. @@ -131,16 +134,16 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Configure the following connection parameters: - - **Server Host**: The IP address or domain name of your TiDB Self-Hosted cluster. - - **Port**: The port number of your TiDB Self-Hosted cluster. - - **Username**: The username to use to connect to your TiDB Self-Hosted cluster. + - **Server Host**: The IP address or domain name of your TiDB Self-Managed cluster. + - **Port**: The port number of your TiDB Self-Managed cluster. + - **Username**: The username to use to connect to your TiDB Self-Managed cluster. - **Password**: The password of the username. An example is as follows: - ![Configure connection settings for TiDB Self-Hosted](/media/develop/dbeaver-connection-settings-self-hosted.jpg) + ![Configure connection settings for TiDB Self-Managed](/media/develop/dbeaver-connection-settings-self-hosted.jpg) -3. Click **Test Connection** to validate the connection to the TiDB Self-Hosted cluster. +3. Click **Test Connection** to validate the connection to the TiDB Self-Managed cluster. If the **Download driver files** dialog is displayed, click **Download** to get the driver files. @@ -163,4 +166,14 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-gui-mysql-workbench.md b/develop/dev-guide-gui-mysql-workbench.md new file mode 100644 index 0000000000000..6943133ffb35f --- /dev/null +++ b/develop/dev-guide-gui-mysql-workbench.md @@ -0,0 +1,185 @@ +--- +title: Connect to TiDB with MySQL Workbench +summary: Learn how to connect to TiDB using MySQL Workbench. +--- + +# Connect to TiDB with MySQL Workbench + +TiDB is a MySQL-compatible database, and [MySQL Workbench](https://www.mysql.com/products/workbench/) is a GUI tool set for MySQL database users. + +> **Warning:** +> +> - Although you can use MySQL Workbench to connect to TiDB due to its MySQL compatibility, MySQL Workbench does not fully support TiDB. You might encounter some issues during usage as it treats TiDB as MySQL. +> - It is recommended to use other GUI tools that officially support TiDB, such as [DataGrip](/develop/dev-guide-gui-datagrip.md), [DBeaver](/develop/dev-guide-gui-dbeaver.md), and [VS Code SQLTools](/develop/dev-guide-gui-vscode-sqltools.md). For a complete list of GUI tools that fully supported by TiDB, see [Third-party tools supported by TiDB](/develop/dev-guide-third-party-support.md#gui). + +In this tutorial, you can learn how to connect to your TiDB cluster using MySQL Workbench. + +> **Note:** +> +> This tutorial is compatible with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed. + +## Prerequisites + +To complete this tutorial, you need: + +- [MySQL Workbench](https://dev.mysql.com/downloads/workbench/) **8.0.31** or later versions. +- A TiDB cluster. + + + +**If you don't have a TiDB cluster, you can create one as follows:** + +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. + + + + +**If you don't have a TiDB cluster, you can create one as follows:** + +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. + + + +## Connect to TiDB + +Connect to your TiDB cluster depending on the TiDB deployment option you have selected. + + +
+ +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Ensure the configurations in the connection dialog match your operating environment. + + - **Connection Type** is set to `Public`. + - **Branch** is set to `main`. + - **Connect With** is set to `MySQL Workbench`. + - **Operating System** matches your environment. + +4. Click **Generate Password** to create a random password. + + > **Tip:** + > + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. + +5. Launch MySQL Workbench and click **+** near the **MySQL Connections** title. + + ![MySQL Workbench: add new connection](/media/develop/mysql-workbench-add-new-connection.png) + +6. In the **Setup New Connection** dialog, configure the following connection parameters: + + - **Connection Name**: give this connection a meaningful name. + - **Hostname**: enter the `HOST` parameter from the TiDB Cloud connection dialog. + - **Port**: enter the `PORT` parameter from the TiDB Cloud connection dialog. + - **Username**: enter the `USERNAME` parameter from the TiDB Cloud connection dialog. + - **Password**: click **Store in Keychain ...** or **Store in Vault**, enter the password of the {{{ .starter }}} cluster, and then click **OK** to store the password. + + ![MySQL Workbench: store the password of {{{ .starter }}} in keychain](/media/develop/mysql-workbench-store-password-in-keychain.png) + + The following figure shows an example of the connection parameters: + + ![MySQL Workbench: configure connection settings for {{{ .starter }}}](/media/develop/mysql-workbench-connection-config-serverless-parameters.png) + +7. Click **Test Connection** to validate the connection to the {{{ .starter }}} cluster. + +8. If the connection test is successful, you can see the **Successfully made the MySQL connection** message. Click **OK** to save the connection configuration. + +
+
+ +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. + + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). + +4. Launch MySQL Workbench and click **+** near the **MySQL Connections** title. + + ![MySQL Workbench: add new connection](/media/develop/mysql-workbench-add-new-connection.png) + +5. In the **Setup New Connection** dialog, configure the following connection parameters: + + - **Connection Name**: give this connection a meaningful name. + - **Hostname**: enter the `HOST` parameter from the TiDB Cloud connection dialog. + - **Port**: enter the `PORT` parameter from the TiDB Cloud connection dialog. + - **Username**: enter the `USERNAME` parameter from the TiDB Cloud connection dialog. + - **Password**: click **Store in Keychain ...**, enter the password of the TiDB Cloud Dedicated cluster, and then click **OK** to store the password. + + ![MySQL Workbench: store the password of TiDB Cloud Dedicated in keychain](/media/develop/mysql-workbench-store-dedicated-password-in-keychain.png) + + The following figure shows an example of the connection parameters: + + ![MySQL Workbench: configure connection settings for TiDB Cloud Dedicated](/media/develop/mysql-workbench-connection-config-dedicated-parameters.png) + +6. Click **Test Connection** to validate the connection to the TiDB Cloud Dedicated cluster. + +7. If the connection test is successful, you can see the **Successfully made the MySQL connection** message. Click **OK** to save the connection configuration. + +
+
+ +1. Launch MySQL Workbench and click **+** near the **MySQL Connections** title. + + ![MySQL Workbench: add new connection](/media/develop/mysql-workbench-add-new-connection.png) + +2. In the **Setup New Connection** dialog, configure the following connection parameters: + + - **Connection Name**: give this connection a meaningful name. + - **Hostname**: enter the IP address or domain name of your TiDB Self-Managed cluster. + - **Port**: enter the port number of your TiDB Self-Managed cluster. + - **Username**: enter the username to use to connect to your TiDB. + - **Password**: click **Store in Keychain ...**, enter the password to use to connect to your TiDB cluster, and then click **OK** to store the password. + + ![MySQL Workbench: store the password of TiDB Self-Managed in keychain](/media/develop/mysql-workbench-store-self-hosted-password-in-keychain.png) + + The following figure shows an example of the connection parameters: + + ![MySQL Workbench: configure connection settings for TiDB Self-Managed](/media/develop/mysql-workbench-connection-config-self-hosted-parameters.png) + +3. Click **Test Connection** to validate the connection to the TiDB Self-Managed cluster. + +4. If the connection test is successful, you can see the **Successfully made the MySQL connection** message. Click **OK** to save the connection configuration. + +
+
+ +## FAQs + +### How to handle the connection timeout error "Error Code: 2013. Lost connection to MySQL server during query"? + +This error indicates that the query execution time exceeds the timeout limit. To resolve this issue, you can adjust the timeout settings by the following steps: + +1. Launch MySQL Workbench and navigate to the **Workbench Preferences** page. +2. In the **SQL Editor** > **MySQL Session** section, configure the **DBMS connection read timeout interval (in seconds)** option. This sets the maximum amount of time (in seconds) that a query can take before MySQL Workbench disconnects from the server. + + ![MySQL Workbench: adjust timeout option in SQL Editor settings](/media/develop/mysql-workbench-adjust-sqleditor-read-timeout.jpg) + +For more information, see [MySQL Workbench frequently asked questions](https://dev.mysql.com/doc/workbench/en/workbench-faq.html). + +## Next steps + +- Learn more usage of MySQL Workbench from [the documentation of MySQL Workbench](https://dev.mysql.com/doc/workbench/en/). +- Learn the best practices for TiDB application development with the chapters in the [Developer guide](/develop/dev-guide-overview.md), such as [Insert data](/develop/dev-guide-insert-data.md), [Update data](/develop/dev-guide-update-data.md), [Delete data](/develop/dev-guide-delete-data.md), [Single table reading](/develop/dev-guide-get-data-from-single-table.md), [Transactions](/develop/dev-guide-transaction-overview.md), and [SQL performance optimization](/develop/dev-guide-optimize-sql-overview.md). +- Learn through the professional [TiDB developer courses](https://www.pingcap.com/education/) and earn [TiDB certifications](https://www.pingcap.com/education/certification/) after passing the exam. + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-gui-navicat.md b/develop/dev-guide-gui-navicat.md new file mode 100644 index 0000000000000..ea23f7a78c337 --- /dev/null +++ b/develop/dev-guide-gui-navicat.md @@ -0,0 +1,165 @@ +--- +title: Connect to TiDB with Navicat +summary: Learn how to connect to TiDB using Navicat. +--- + +# Connect to TiDB with Navicat + +TiDB is a MySQL-compatible database, and [Navicat](https://www.navicat.com) is a GUI tool set for database users. This tutorial uses the [Navicat Premium](https://www.navicat.com/en/products/navicat-premium) tool to connect to TiDB. + +In this tutorial, you can learn how to connect to your TiDB cluster using Navicat. + +> **Note:** +> +> This tutorial is compatible with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed. + +## Prerequisites + +To complete this tutorial, you need: + +- [Navicat Premium](https://www.navicat.com) **17.1.6** or later versions. +- A paid account for Navicat Premium. +- A TiDB cluster. + + + +**If you don't have a TiDB cluster, you can create one as follows:** + +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. + + + + +**If you don't have a TiDB cluster, you can create one as follows:** + +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. + + + +## Connect to TiDB + +Connect to your TiDB cluster depending on the TiDB deployment option you have selected. + + +
+ +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Ensure the configurations in the connection dialog match your operating environment. + + - **Connection Type** is set to `Public`. + - **Branch** is set to `main`. + - **Connect With** is set to `Navicat`. + - **Operating System** matches your environment. + +4. Click **Generate Password** to create a random password. + + > **Tip:** + > + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. + +5. Launch Navicat Premium, click **Connection** in the upper-left corner, select **PingCAP** from the **Venfor Filter** list, and double-click **TiDB** in the right panel. + + ![Navicat: add new connection](/media/develop/navicat-premium-add-new-connection.png) + +6. In the **New Connection (TiDB)** dialog, configure the following connection parameters: + + - **Connection Name**: give this connection a meaningful name. + - **Host**: enter the `HOST` parameter from the TiDB Cloud connection dialog. + - **Port**: enter the `PORT` parameter from the TiDB Cloud connection dialog. + - **User Name**: enter the `USERNAME` parameter from the TiDB Cloud connection dialog. + - **Password**: enter the password of the {{{ .starter }}} cluster. + + ![Navicat: configure connection general panel for {{{ .starter }}}](/media/develop/navicat-premium-connection-config-serverless-general.png) + +7. Click the **SSL** tab and select **Use SSL**, **Use authentication**, and **Verify server certificate against CA** checkboxes. Then, select the `CA` file from the TiDB Cloud connection dialog into the **CA Certificate** field. + + ![Navicat: configure connection SSL panel for {{{ .starter }}}](/media/develop/navicat-premium-connection-config-serverless-ssl.png) + +8. Click **Test Connection** to validate the connection to the {{{ .starter }}} cluster. + +9. If the connection test is successful, you can see the **Connection Successful** message. Click **OK** to finish the connection configuration. + +
+
+ +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list. + + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). + +4. Click **CA cert** to download the CA certificate. + +5. Launch Navicat Premium, click **Connection** in the upper-left corner, select **PingCAP** from the **Venfor Filter** list, and double-click **TiDB** in the right panel. + + ![Navicat: add new connection](/media/develop/navicat-premium-add-new-connection.png) + +6. In the **New Connection (TiDB)** dialog, configure the following connection parameters: + + - **Connection Name**: give this connection a meaningful name. + - **Host**: enter the `HOST` parameter from the TiDB Cloud connection dialog. + - **Port**: enter the `PORT` parameter from the TiDB Cloud connection dialog. + - **User Name**: enter the `USERNAME` parameter from the TiDB Cloud connection dialog. + - **Password**: enter the password of the TiDB Cloud Dedicated cluster. + + ![Navicat: configure connection general panel for TiDB Cloud Dedicated](/media/develop/navicat-premium-connection-config-dedicated-general.png) + +7. Click the **SSL** tab and select **Use SSL**, **Use authentication**, and **Verify server certificate against CA** checkboxes. Then, select the CA file downloaded in step 4 into the **CA Certificate** field. + + ![Navicat: configure connection SSL panel for TiDB Cloud Dedicated](/media/develop/navicat-premium-connection-config-dedicated-ssl.png) + +8. **Test Connection** to validate the connection to the TiDB Cloud Dedicated cluster. + +9. If the connection test is successful, you can see the **Connection Successful** message. Click **OK** to finish the connection configuration. + +
+
+ +1. Launch Navicat Premium, click **Connection** in the upper-left corner, select **PingCAP** from the **Venfor Filter** list, and double-click **TiDB** in the right panel. + + ![Navicat: add new connection](/media/develop/navicat-premium-add-new-connection.png) + +2. In the **New Connection (TiDB)** dialog, configure the following connection parameters: + + - **Connection Name**: give this connection a meaningful name. + - **Host**: enter the IP address or domain name of your TiDB Self-Managed cluster. + - **Port**: enter the port number of your TiDB Self-Managed cluster. + - **User Name**: enter the username to use to connect to your TiDB. + - **Password**: enter the password to use to connect to your TiDB. + + ![Navicat: configure connection general panel for self-hosted TiDB](/media/develop/navicat-premium-connection-config-self-hosted-general.png) + +3. Click **Test Connection** to validate the connection to the TiDB Self-Managed cluster. + +4. If the connection test is successful, you can see the **Connection Successful** message. Click **OK** to finish the connection configuration. + +
+
+ +## Next steps + +- Learn the best practices for TiDB application development with the chapters in the [Developer guide](/develop/dev-guide-overview.md), such as [Insert data](/develop/dev-guide-insert-data.md), [Update data](/develop/dev-guide-update-data.md), [Delete data](/develop/dev-guide-delete-data.md), [Single table reading](/develop/dev-guide-get-data-from-single-table.md), [Transactions](/develop/dev-guide-transaction-overview.md), and [SQL performance optimization](/develop/dev-guide-optimize-sql-overview.md). +- Learn through the professional [TiDB developer courses](https://www.pingcap.com/education/) and earn [TiDB certifications](https://www.pingcap.com/education/certification/) after passing the exam. + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-gui-vscode-sqltools.md b/develop/dev-guide-gui-vscode-sqltools.md index bc2100505c1f3..a3b2fa03e6a8b 100644 --- a/develop/dev-guide-gui-vscode-sqltools.md +++ b/develop/dev-guide-gui-vscode-sqltools.md @@ -11,7 +11,7 @@ In this tutorial, you can learn how to connect to your TiDB cluster using Visual > **Note:** > -> - This tutorial is compatible with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> - This tutorial is compatible with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed. > - This tutorial also works with Visual Studio Code Remote Development environments, such as [GitHub Codespaces](https://github.com/features/codespaces), [Visual Studio Code Dev Containers](https://code.visualstudio.com/docs/devcontainers/containers), and [Visual Studio Code WSL](https://code.visualstudio.com/docs/remote/wsl). ## Prerequisites @@ -22,13 +22,14 @@ To complete this tutorial, you need: - [SQLTools MySQL/MariaDB/TiDB](https://marketplace.visualstudio.com/items?itemName=mtxr.sqltools-driver-mysql) extension for Visual Studio Code. To install it, you can use one of the following methods: - Click this link to launch VS Code and install the extension directly. - Navigate to [VS Code Marketplace](https://marketplace.visualstudio.com/items?itemName=mtxr.sqltools-driver-mysql) and click **Install**. + - On the **Extensions** tab of your VS Code, search for `mtxr.sqltools-driver-mysql` to get the **SQLTools MySQL/MariaDB/TiDB** extension, and then click **Install**. - A TiDB cluster. **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -36,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -46,7 +47,7 @@ To complete this tutorial, you need: Connect to your TiDB cluster depending on the TiDB deployment option you have selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -54,19 +55,20 @@ Connect to your TiDB cluster depending on the TiDB deployment option you have se 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public`. - - **Connect With** is set to `General`. + - **Connection Type** is set to `Public`. + - **Branch** is set to `main`. + - **Connect With** is set to `VS Code`. - **Operating System** matches your environment. > **Tip:** > > If your VS Code is running on a remote development environment, select the remote operating system from the list. For example, if you are using Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. This is not necessary if you are using GitHub Codespaces. -4. Click **Create password** to create a random password. +4. Click **Generate Password** to create a random password. > **Tip:** > - > If you have created a password before, you can either use the original password or click **Reset password** to generate a new one. + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. 5. Launch VS Code and select the **SQLTools** extension on the navigation pane. Under the **CONNECTIONS** section, click **Add New Connection** and select **TiDB** as the database driver. @@ -77,41 +79,43 @@ Connect to your TiDB cluster depending on the TiDB deployment option you have se - **Connection name**: give this connection a meaningful name. - **Connection group**: (optional) give this group of connections a meaningful name. Connections with the same group name will be grouped together. - **Connect using**: select **Server and Port**. - - **Server Address**: enter the `host` parameter from the TiDB Cloud connection dialog. - - **Port**: enter the `port` parameter from the TiDB Cloud connection dialog. + - **Server Address**: enter the `HOST` parameter from the TiDB Cloud connection dialog. + - **Port**: enter the `PORT` parameter from the TiDB Cloud connection dialog. - **Database**: enter the database that you want to connect to. - - **Username**: enter the `user` parameter from the TiDB Cloud connection dialog. + - **Username**: enter the `USERNAME` parameter from the TiDB Cloud connection dialog. - **Password mode**: select **SQLTools Driver Credentials**. - In the **MySQL driver specific options** area, configure the following parameters: - **Authentication Protocol**: select **default**. - - **SSL**: select **Enabled**. TiDB Serverless requires a secure connection. In the **SSL Options (node.TLSSocket)** area, configure the **Certificate Authority (CA) Certificate File** field as the `ssl_ca` parameter from the TiDB Cloud connection dialog. + - **SSL**: select **Enabled**. {{{ .starter }}} requires a secure connection. In the **SSL Options (node.TLSSocket)** area, configure the **Certificate Authority (CA) Certificate File** field as the `CA` parameter from the TiDB Cloud connection dialog. > **Note:** > - > If you are running on Windows or GitHub Codespaces, you can leave **SSL** blank. By default SQLTools trusts well-known CAs curated by Mozilla. For more information, see [TiDB Serverless root certificate management](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters#root-certificate-management). + > If you are running on Windows or GitHub Codespaces, you can leave **SSL** blank. By default SQLTools trusts well-known CAs curated by Let's Encrypt. For more information, see [{{{ .starter }}} root certificate management](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters#root-certificate-management). - ![VS Code SQLTools: configure connection settings for TiDB Serverless](/media/develop/vsc-sqltools-connection-config-serverless.jpg) + ![VS Code SQLTools: configure connection settings for {{{ .starter }}}](/media/develop/vsc-sqltools-connection-config-serverless.jpg) -7. Click **TEST CONNECTION** to validate the connection to the TiDB Serverless cluster. +7. Click **TEST CONNECTION** to validate the connection to the {{{ .starter }}} cluster. 1. In the pop-up window, click **Allow**. 2. In the **SQLTools Driver Credentials** dialog, enter the password you created in step 4. - ![VS Code SQLTools: enter password to connect to TiDB Serverless](/media/develop/vsc-sqltools-password.jpg) + ![VS Code SQLTools: enter password to connect to {{{ .starter }}}](/media/develop/vsc-sqltools-password.jpg) 8. If the connection test is successful, you can see the **Successfully connected!** message. Click **SAVE CONNECTION** to save the connection configuration.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere**. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Launch VS Code and select the **SQLTools** extension on the navigation pane. Under the **CONNECTIONS** section, click **Add New Connection** and select **TiDB** as the database driver. @@ -132,19 +136,19 @@ Connect to your TiDB cluster depending on the TiDB deployment option you have se - **Authentication Protocol**: select **default**. - **SSL**: select **Disabled**. - ![VS Code SQLTools: configure connection settings for TiDB Dedicated](/media/develop/vsc-sqltools-connection-config-dedicated.jpg) + ![VS Code SQLTools: configure connection settings for TiDB Cloud Dedicated](/media/develop/vsc-sqltools-connection-config-dedicated.jpg) -6. Click **TEST CONNECTION** to validate the connection to the TiDB Dedicated cluster. +6. Click **TEST CONNECTION** to validate the connection to the TiDB Cloud Dedicated cluster. 1. In the pop-up window, click **Allow**. - 2. In the **SQLTools Driver Credentials** dialog, enter the password of the TiDB Dedicated cluster. + 2. In the **SQLTools Driver Credentials** dialog, enter the password of the TiDB Cloud Dedicated cluster. - ![VS Code SQLTools: enter password to connect to TiDB Dedicated](/media/develop/vsc-sqltools-password.jpg) + ![VS Code SQLTools: enter password to connect to TiDB Cloud Dedicated](/media/develop/vsc-sqltools-password.jpg) 7. If the connection test is successful, you can see the **Successfully connected!** message. Click **SAVE CONNECTION** to save the connection configuration.
-
+
1. Launch VS Code and select the **SQLTools** extension on the navigation pane. Under the **CONNECTIONS** section, click **Add New Connection** and select **TiDB** as the database driver. @@ -155,10 +159,10 @@ Connect to your TiDB cluster depending on the TiDB deployment option you have se - **Connection name**: give this connection a meaningful name. - **Connection group**: (optional) give this group of connections a meaningful name. Connections with the same group name will be grouped together. - **Connect using**: select **Server and Port**. - - **Server Address**: enter the IP address or domain name of your TiDB Self-Hosted cluster. - - **Port**: enter the port number of your TiDB Self-Hosted cluster. + - **Server Address**: enter the IP address or domain name of your TiDB Self-Managed cluster. + - **Port**: enter the port number of your TiDB Self-Managed cluster. - **Database**: enter the database that you want to connect to. - - **Username**: enter the username to use to connect to your TiDB Self-Hosted cluster. + - **Username**: enter the username to use to connect to your TiDB Self-Managed cluster. - **Password mode**: - If the password is empty, select **Use empty password**. @@ -169,13 +173,13 @@ Connect to your TiDB cluster depending on the TiDB deployment option you have se - **Authentication Protocol**: select **default**. - **SSL**: select **Disabled**. - ![VS Code SQLTools: configure connection settings for TiDB Self-Hosted](/media/develop/vsc-sqltools-connection-config-self-hosted.jpg) + ![VS Code SQLTools: configure connection settings for TiDB Self-Managed](/media/develop/vsc-sqltools-connection-config-self-hosted.jpg) -3. Click **TEST CONNECTION** to validate the connection to the TiDB Self-Hosted cluster. +3. Click **TEST CONNECTION** to validate the connection to the TiDB Self-Managed cluster. - If the password is not empty, click **Allow** in the pop-up window, and then enter the password of the TiDB Self-Hosted cluster. + If the password is not empty, click **Allow** in the pop-up window, and then enter the password of the TiDB Self-Managed cluster. - ![VS Code SQLTools: enter password to connect to TiDB Self-Hosted](/media/develop/vsc-sqltools-password.jpg) + ![VS Code SQLTools: enter password to connect to TiDB Self-Managed](/media/develop/vsc-sqltools-password.jpg) 4. If the connection test is successful, you can see the **Successfully connected!** message. Click **SAVE CONNECTION** to save the connection configuration. @@ -191,4 +195,14 @@ Connect to your TiDB cluster depending on the TiDB deployment option you have se ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-hybrid-oltp-and-olap-queries.md b/develop/dev-guide-hybrid-oltp-and-olap-queries.md index 7e6a31c0b0f79..1d1c6528700c8 100644 --- a/develop/dev-guide-hybrid-oltp-and-olap-queries.md +++ b/develop/dev-guide-hybrid-oltp-and-olap-queries.md @@ -261,3 +261,17 @@ For more information about how TiDB chooses to use TiFlash, see [Use TiDB to rea - [Window Functions](/functions-and-operators/window-functions.md) - [Use TiFlash](/tiflash/tiflash-overview.md#use-tiflash) + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-implicit-type-conversion.md b/develop/dev-guide-implicit-type-conversion.md index d801fb2147af6..4462ec600f49c 100644 --- a/develop/dev-guide-implicit-type-conversion.md +++ b/develop/dev-guide-implicit-type-conversion.md @@ -76,3 +76,17 @@ SELECT * FROM `t1` WHERE `a` BETWEEN '12123123' AND '1111222211111111200000'; ``` **Brief description of run results**: The above execution gives a wrong result. + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-index-best-practice.md b/develop/dev-guide-index-best-practice.md index 1445cde3b2778..7d242bde1c8cd 100644 --- a/develop/dev-guide-index-best-practice.md +++ b/develop/dev-guide-index-best-practice.md @@ -150,3 +150,17 @@ CREATE TABLE `books` ( ``` - When using the `IN` expression in a query condition, it is recommended that the number of value matched after it does not exceed 300, otherwise the execution efficiency will be poor. + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-insert-data.md b/develop/dev-guide-insert-data.md index e8ab1f17b1b8a..3a3469d018c95 100644 --- a/develop/dev-guide-insert-data.md +++ b/develop/dev-guide-insert-data.md @@ -13,7 +13,7 @@ This document describes how to insert data into TiDB by using the SQL language w Before reading this document, you need to prepare the following: -- [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). +- [Build a {{{ .starter }}} Cluster](/develop/dev-guide-build-cluster-in-cloud.md). - Read [Schema Design Overview](/develop/dev-guide-schema-design-overview.md), [Create a Database](/develop/dev-guide-create-database.md), [Create a Table](/develop/dev-guide-create-table.md), and [Create Secondary Indexes](/develop/dev-guide-create-secondary-indexes.md) ## Insert rows @@ -85,7 +85,7 @@ Due to the default MySQL JDBC Driver settings, you need to change some parameter | Parameter | Means | Recommended Scenario | Recommended Configuration| | :------------------------: | :-----------------------------------: | :-----------------------------------------------------------------------------------------------------------------------------------------------: | :----------------------: | | `useServerPrepStmts` | Whether to use the server side to enable prepared statements | When you need to use a prepared statement more than once | `true` | -| `cachePrepStmts` | Whether the client caches prepared statements | `useServerPrepStmts=true` 时 | `true` | +| `cachePrepStmts` | Whether the client caches prepared statements | `useServerPrepStmts=true` | `true` | | `prepStmtCacheSqlLimit` | Maximum size of a prepared statement (256 characters by default) | When the prepared statement is greater than 256 characters | Configured according to the actual size of the prepared statement | | `prepStmtCacheSize` | Maximum number of prepared statement caches (25 by default) | When the number of prepared statements is greater than 25 | Configured according to the actual number of prepared statements | | `rewriteBatchedStatements` | Whether to rewrite **Batched** statements | When batch operations are required | `true` | @@ -290,7 +290,7 @@ There are two solutions to handle this error: INSERT INTO `bookshop`.`users` (`balance`, `nickname`) VALUES (0.00, 'nicky'); ``` -- If you are sure that you **_must_** specify this column, then you can use the [`SET` statement](https://docs.pingcap.com/zh/tidb/stable/sql-statement-set-variable) to allow the column of `AUTO_RANDOM` to be specified during insertion time by changing the user variable. +- If you are sure that you **_must_** specify this column, then you can use the [`SET` statement](https://docs.pingcap.com/tidb/stable/sql-statement-set-variable) to allow the column of `AUTO_RANDOM` to be specified during insertion time by changing the user variable. {{< copyable "sql" >}} @@ -302,3 +302,17 @@ There are two solutions to handle this error: ## Use HTAP In TiDB, HTAP capabilities save you from performing additional operations when inserting data. There is no additional insertion logic. TiDB automatically guarantees data consistency. All you need to do is [turn on column-oriented replica synchronization](/develop/dev-guide-create-table.md#use-htap-capabilities) after creating the table, and use the column-oriented replica to speed up your queries directly. + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-join-tables.md b/develop/dev-guide-join-tables.md index e22b59bb4ccb5..b23290b0f355c 100644 --- a/develop/dev-guide-join-tables.md +++ b/develop/dev-guide-join-tables.md @@ -253,3 +253,17 @@ For more information about the implementation details and limitations of this Jo - [Explain Statements That Use Joins](/explain-joins.md) - [Introduction to Join Reorder](/join-reorder.md) + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-object-naming-guidelines.md b/develop/dev-guide-object-naming-guidelines.md index 04e03b8202b99..8fb6f3adf8c6c 100644 --- a/develop/dev-guide-object-naming-guidelines.md +++ b/develop/dev-guide-object-naming-guidelines.md @@ -43,4 +43,18 @@ It is recommended to differentiate database names by business, product, or other - Primary key index: `pk_{table_name_abbreviation}_{field_name_abbreviation}` - Unique index: `uk_{table_name_abbreviation}_{field_name_abbreviation}` - Common index: `idx_{table_name_abbreviation}_{field_name_abbreviation}` -- Column name with multiple words: use meaningful abbreviations \ No newline at end of file +- Column name with multiple words: use meaningful abbreviations + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-optimistic-and-pessimistic-transaction.md b/develop/dev-guide-optimistic-and-pessimistic-transaction.md index b20a2be9e3f7c..5b3361b052532 100644 --- a/develop/dev-guide-optimistic-and-pessimistic-transaction.md +++ b/develop/dev-guide-optimistic-and-pessimistic-transaction.md @@ -1365,4 +1365,18 @@ mysql> SELECT * FROM users; | 2 | 9600.00 | Alice | +----+----------+----------+ 2 rows in set (0.00 sec) -``` \ No newline at end of file +``` + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-optimize-sql-best-practices.md b/develop/dev-guide-optimize-sql-best-practices.md index 6463e68195477..1f3a62262fbc7 100644 --- a/develop/dev-guide-optimize-sql-best-practices.md +++ b/develop/dev-guide-optimize-sql-best-practices.md @@ -167,7 +167,7 @@ For how to locate and resolve transaction conflicts, see [Troubleshoot Lock Conf -See [Best Practices for Developing Java Applications with TiDB](/best-practices/java-app-best-practices.md). +See [Best Practices for Developing Java Applications with TiDB](/develop/java-app-best-practices.md). @@ -190,3 +190,17 @@ See [Best Practices for Developing Java Applications with TiDB](https://docs.pin - [Highly Concurrent Write Best Practices](https://docs.pingcap.com/tidb/stable/high-concurrency-best-practices) + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-optimize-sql-overview.md b/develop/dev-guide-optimize-sql-overview.md index b3468ba51a505..029aa1eeed791 100644 --- a/develop/dev-guide-optimize-sql-overview.md +++ b/develop/dev-guide-optimize-sql-overview.md @@ -50,3 +50,17 @@ After [tuning SQL performance](#sql-performance-tuning), if your application sti * [SQL Performance Tuning](/tidb-cloud/tidb-cloud-sql-tuning-overview.md) + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-optimize-sql.md b/develop/dev-guide-optimize-sql.md index 8651eff3452df..ce767696c9f66 100644 --- a/develop/dev-guide-optimize-sql.md +++ b/develop/dev-guide-optimize-sql.md @@ -244,3 +244,17 @@ See [JOIN Execution Plan](/explain-joins.md). * [EXPLAIN Walkthrough](/explain-walkthrough.md) * [Explain Statements That Use Indexes](/explain-indexes.md) + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-overview.md b/develop/dev-guide-overview.md index 631d238aa2a9e..a5aa14f38e4cd 100644 --- a/develop/dev-guide-overview.md +++ b/develop/dev-guide-overview.md @@ -1,27 +1,193 @@ --- title: Developer Guide Overview -summary: Introduce the overview of the developer guide. -aliases: ['/tidb/dev/connectors-and-apis/','/appdev/dev/','/tidb/dev/dev-guide-outdated-for-laravel'] +summary: Introduce the overview of the developer guide for TiDB Cloud and TiDB Self-Managed. --- # Developer Guide Overview -This guide is written for application developers, but if you are interested in the inner workings of TiDB or want to get involved in TiDB development, read the [TiDB Kernel Development Guide](https://pingcap.github.io/tidb-dev-guide/) for more information about TiDB. + - + + + -This tutorial shows how to quickly build an application using TiDB, the possible use cases of TiDB and how to handle common problems. +## Guides by language and framework -Before reading this page, it is recommended that you read the [Quick Start Guide for the TiDB Database Platform](/quick-start-with-tidb.md). +Build your application with the language you use by following the guides with sample codes. - + + - +Connect to TiDB Cloud over HTTPS in Edge Function. + + + + +Connect Next.js with mysql2 to TiDB Cloud. + + + + +Connect to TiDB Cloud with Prisma ORM. + + + + +Connect to TiDB Cloud with TypeORM. + + + + +Connect to TiDB Cloud with Sequelize ORM. + + + + +Connect Node.js with mysql.js module to TiDB Cloud. + + + + +Connect Node.js with node-mysql2 module to TiDB Cloud. + + + + +Connect AWS Lambda Function with mysql2 to TiDB Cloud. + + + + + + + +Connect Django application with django-tidb to TiDB Cloud. + + + + +Connect to TiDB Cloud with MySQL official package. + + + + +Connect to TiDB Cloud with PyMySQL package. + + + + +Connect to TiDB Cloud with mysqlclient package. + + + + +Connect to TiDB Cloud with SQLAlchemy ORM. + + + + +Connect to TiDB Cloud with Peewee ORM. + + + + + + + +Connect to TiDB Cloud with JDBC (MySQL Connector/J). + + + + +Connect to TiDB Cloud with MyBatis ORM. + + + + +Connect to TiDB Cloud with Hibernate ORM. + + + + +Connect Spring based application with Spring Data JPA to TiDB Cloud. + + + + + + + +Connect to TiDB Cloud with MySQL driver for Go. + + + + +Connect to TiDB Cloud with GORM. + + + + + + + +Connect Ruby on Rails application with Active Record ORM to TiDB Cloud. + + + + +Connect to TiDB Cloud with mysql2 driver. -This tutorial shows how to quickly build an application using TiDB Cloud, the possible use cases of TiDB Cloud and how to handle common problems. + + + +In addition to these guides, PingCAP works with the community to [support the third-party MySQL driver, ORMs and tools](/develop/dev-guide-third-party-support.md). + +## Use MySQL client software + +As TiDB is a MySQL-compatible database, you can use many client software tools to connect to TiDB Cloud and manage the databases just like you did before. Or, use our command line tool to connect and manage your databases. + + + + +Connect and manage TiDB Cloud databases with MySQL Workbench. + + + + +Connect and manage TiDB Cloud databases with SQLTools extension in VSCode. + + + + +Connect and manage TiDB Cloud databases with DBeaver. + + + + +Connect and manage TiDB Cloud databases with DataGrip by JetBrains. + + + + +## Additional resources + +Learn other topics about developing with TiDB Cloud. + +- Use TiDB Cloud CLI to develop, manage and deploy your applications. +- Explore popular services integrations with TiDB Cloud. +- Use [TiDB database development reference](/develop/dev-guide-schema-design-overview.md) to design, interact, optimize and troubleshoot with your data and schema. +- Follow the free online course [Introduction to TiDB](https://eng.edu.pingcap.com/catalog/info/id:203/?utm_source=docs-dev-guide). + + +This guide is written for application developers, but if you are interested in the inner workings of TiDB or want to get involved in TiDB development, read the [TiDB Kernel Development Guide](https://pingcap.github.io/tidb-dev-guide/) for more information about TiDB. + +This tutorial shows how to quickly build an application using TiDB, the possible use cases of TiDB and how to handle common problems. + +Before reading this page, it is recommended that you read the [Quick Start Guide for the TiDB Database Platform](/quick-start-with-tidb.md). + ## TiDB basics Before you start working with TiDB, you need to understand some important mechanisms of how TiDB works: @@ -38,18 +204,8 @@ You can start a transaction using [`BEGIN`](/sql-statements/sql-statement-begin. TiDB guarantees atomicity for all statements between the start of `BEGIN` and the end of `COMMIT` or `ROLLBACK`, that is, all statements that are executed during this period either succeed or fail as a whole. This is used to ensure data consistency you need for application development. - - If you are not sure what an **optimistic transaction** is, do **_NOT_** use it yet. Because **optimistic transactions** require that the application can correctly handle [all errors](/error-codes.md) returned by the `COMMIT` statement. If you are not sure how your application handles them, use a **pessimistic transaction** instead. - - - - -If you are not sure what an **optimistic transaction** is, do **_NOT_** use it yet. Because **optimistic transactions** require that the application can correctly handle [all errors](https://docs.pingcap.com/tidb/stable/error-codes) returned by the `COMMIT` statement. If you are not sure how your application handles them, use a **pessimistic transaction** instead. - - - ## The way applications interact with TiDB TiDB is highly compatible with the MySQL protocol and supports [most MySQL syntax and features](/mysql-compatibility.md), so most MySQL connection libraries are compatible with TiDB. If your application framework or language does not have an official adaptation from PingCAP, it is recommended that you use MySQL's client libraries. More and more third-party libraries are actively supporting TiDB's different features. @@ -58,8 +214,6 @@ Since TiDB is compatible with the MySQL protocol and MySQL syntax, most of the O ## Read more - - - [Quick Start](/develop/dev-guide-build-cluster-in-cloud.md) - [Choose Driver or ORM](/develop/dev-guide-choose-driver-or-orm.md) - [Connect to TiDB](/develop/dev-guide-connect-to-tidb.md) @@ -70,34 +224,8 @@ Since TiDB is compatible with the MySQL protocol and MySQL syntax, most of the O - [Optimize](/develop/dev-guide-optimize-sql-overview.md) - [Example Applications](/develop/dev-guide-sample-application-java-spring-boot.md) - - - - -Here you can find additional resources to connect, manage and develop with TiDB Cloud. - -**To explore your data** +## Need help? -- [Quick Start](/develop/dev-guide-build-cluster-in-cloud.md) -- [Use AI-powered SQL Editor beta](/tidb-cloud/explore-data-with-chat2query.md) -- Connect with client tools such as [VSCode](/develop/dev-guide-gui-vscode-sqltools.md), [DBeaver](/develop/dev-guide-gui-dbeaver.md) or [DataGrip](/develop/dev-guide-gui-datagrip.md) - -**To build your application** - -- [Choose Driver or ORM](/develop/dev-guide-choose-driver-or-orm.md) -- [Use TiDB Cloud Data API beta](/tidb-cloud/data-service-overview.md) - -**To manage your cluster** - -- [TiDB Cloud Command Line Tools](/tidb-cloud/get-started-with-cli.md) -- [TiDB Cloud Administration API](https://docs.pingcap.com/tidbcloud/api/v1beta1) - -**To learn more about TiDB** - -- [Database Schema Design](/develop/dev-guide-schema-design-overview.md) -- [Write Data](/develop/dev-guide-insert-data.md) -- [Read Data](/develop/dev-guide-get-data-from-single-table.md) -- [Transaction](/develop/dev-guide-transaction-overview.md) -- [Optimize](/develop/dev-guide-optimize-sql-overview.md) +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). diff --git a/develop/dev-guide-paginate-results.md b/develop/dev-guide-paginate-results.md index f10f432033256..2cdb861e8c642 100644 --- a/develop/dev-guide-paginate-results.md +++ b/develop/dev-guide-paginate-results.md @@ -317,3 +317,17 @@ WHERE ) ORDER BY book_id, user_id; ``` + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-playground-gitpod.md b/develop/dev-guide-playground-gitpod.md index 3d573b66346f0..6daa910bc56d2 100644 --- a/develop/dev-guide-playground-gitpod.md +++ b/develop/dev-guide-playground-gitpod.md @@ -1,5 +1,6 @@ --- title: Gitpod +summary: Gitpod provides a complete, automated, and pre-configured cloud-native development environment. You can develop, run, and test code directly in the browser without any local configurations. --- @@ -32,7 +33,7 @@ After that, you will see a page similar to the following: ![playground gitpod workspace init](/media/develop/playground-gitpod-workspace-init.png) -This scenario in the page uses [TiUP](https://docs.pingcap.com/zh/tidb/stable/tiup-overview) to build a TiDB Playground. You can check the progress on the left side of the terminal area. +This scenario in the page uses [TiUP](https://docs.pingcap.com/tidb/stable/tiup-overview) to build a TiDB Playground. You can check the progress on the left side of the terminal area. Once the TiDB Playground is ready, another `Spring JPA Hibernate` task will run. You can check the progress on the right side of the terminal area. @@ -165,3 +166,17 @@ Visit `https://gitpod.io/workspaces` for all established workspaces. Gitpod provides a complete, automated, and pre-configured cloud-native development environment. You can develop, run, and test code directly in the browser without any local configurations. ![playground gitpod summary](/media/develop/playground-gitpod-summary.png) + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-prepared-statement.md b/develop/dev-guide-prepared-statement.md index deac8b0424c94..859356e6b96d9 100644 --- a/develop/dev-guide-prepared-statement.md +++ b/develop/dev-guide-prepared-statement.md @@ -203,7 +203,7 @@ The following configurations help you use the TiDB server-side prepared statemen | Parameter | Means | Recommended Scenario | Recommended Configuration| | :------------------------: | :-----------------------------------: | :-----------------------------------------------------------------------------------------------------------------------------------------------: | :----------------------: | | `useServerPrepStmts` | Whether to use the server side to enable prepared statements | When you need to use a prepared statement more than once | `true` | -| `cachePrepStmts` | Whether the client caches prepared statements | `useServerPrepStmts=true` 时 | `true` | +| `cachePrepStmts` | Whether the client caches prepared statements | `useServerPrepStmts=true` | `true` | | `prepStmtCacheSqlLimit` | Maximum size of a prepared statement (256 characters by default) | When the prepared statement is greater than 256 characters | Configured according to the actual size of the prepared statement | | `prepStmtCacheSize` | Maximum number of prepared statements (25 by default) | When the number of prepared statements is greater than 25 | Configured according to the actual number of prepared statements | @@ -224,3 +224,17 @@ For a complete example in Java, see:
+ +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-proxysql-integration.md b/develop/dev-guide-proxysql-integration.md index e0465d329a0b4..0fad76da04ed8 100644 --- a/develop/dev-guide-proxysql-integration.md +++ b/develop/dev-guide-proxysql-integration.md @@ -119,11 +119,11 @@ systemctl start docker ### Option 1: Integrate TiDB Cloud with ProxySQL -For this integration, you will be using the [ProxySQL Docker image](https://hub.docker.com/r/proxysql/proxysql) along with a TiDB Serverless cluster. The following steps will set up ProxySQL on port `16033`, so make sure this port is available. +For this integration, you will be using the [ProxySQL Docker image](https://hub.docker.com/r/proxysql/proxysql) along with a {{{ .starter }}} cluster. The following steps will set up ProxySQL on port `16033`, so make sure this port is available. -#### Step 1. Create a TiDB Serverless cluster +#### Step 1. Create a {{{ .starter }}} cluster -1. [Create a free TiDB Serverless cluster](https://docs.pingcap.com/tidbcloud/tidb-cloud-quickstart#step-1-create-a-tidb-cluster). Remember the root password that you set for your cluster. +1. [Create a free {{{ .starter }}} cluster](https://docs.pingcap.com/tidbcloud/tidb-cloud-quickstart#step-1-create-a-tidb-cluster). Remember the root password that you set for your cluster. 2. Get your cluster hostname, port, and username for later use. 1. On the [Clusters](https://tidbcloud.com/console/clusters) page, click your cluster name to go to the cluster overview page. @@ -327,12 +327,12 @@ For this integration, you will be using the [ProxySQL Docker image](https://hub. > > 1. Adds a user using the username and password of your cluster. > 2. Assigns the user to the monitoring account. - > 3. Adds your TiDB Serverless cluster to the list of hosts. - > 4. Enables a secure connection between ProxySQL and the TiDB Serverless cluster. + > 3. Adds your {{{ .starter }}} cluster to the list of hosts. + > 4. Enables a secure connection between ProxySQL and the {{{ .starter }}} cluster. > > To have a better understanding, it is strongly recommended that you check the `proxysql-prepare.sql` file. To learn more about ProxySQL configuration, see [ProxySQL documentation](https://proxysql.com/documentation/proxysql-configuration/). - The following is an example output. You will see that the hostname of your cluster is shown in the output, which means that the connectivity between ProxySQL and the TiDB Serverless cluster is established. + The following is an example output. You will see that the hostname of your cluster is shown in the output, which means that the connectivity between ProxySQL and the {{{ .starter }}} cluster is established. ``` *************************** 1. row *************************** @@ -388,7 +388,7 @@ For this integration, you will be using the [ProxySQL Docker image](https://hub. SELECT VERSION(); ``` - If the TiDB version is displayed, you are successfully connected to your TiDB Serverless cluster through ProxySQL. To exit from the MySQL client anytime, enter `quit` and press enter. + If the TiDB version is displayed, you are successfully connected to your {{{ .starter }}} cluster through ProxySQL. To exit from the MySQL client anytime, enter `quit` and press enter. > **Note:** > @@ -624,7 +624,7 @@ The following steps will set up ProxySQL and TiDB on ports `6033` and `4000` res ## Production environment -For a production environment, it is recommended that you use [TiDB Cloud](https://en.pingcap.com/tidb-cloud/) directly for a fully-managed experience. +For a production environment, it is recommended that you use [TiDB Cloud Dedicated](https://www.pingcap.com/tidb-dedicated/) directly for a fully-managed experience. ### Prerequisite @@ -636,7 +636,7 @@ ProxySQL can be installed on many different platforms. The following takes CentO For a full list of supported platforms and the corresponding version requirements, see [ProxySQL documentation](https://proxysql.com/documentation/installing-proxysql/). -#### Step 1. Create a TiDB Dedicated cluster +#### Step 1. Create a TiDB Cloud Dedicated cluster For detailed steps, see [Create a TiDB Cluster](https://docs.pingcap.com/tidbcloud/create-tidb-cluster). @@ -687,7 +687,7 @@ To use ProxySQL as a proxy for TiDB, you need to configure ProxySQL. To do so, y The above step will take you to the ProxySQL admin prompt. -2. Configure the TiDB clusters to be used, where you can add one or multiple TiDB clusters to ProxySQL. The following statement will add one TiDB Dedicated cluster for example. You need to replace `` and `` with your TiDB Cloud endpoint and port (the default port is `4000`). +2. Configure the TiDB clusters to be used, where you can add one or multiple TiDB clusters to ProxySQL. The following statement will add one TiDB Cloud Dedicated cluster for example. You need to replace `` and `` with your TiDB Cloud endpoint and port (the default port is `4000`). ```sql INSERT INTO mysql_servers(hostgroup_id, hostname, port) @@ -1124,4 +1124,18 @@ Databases can be overloaded by high traffic, faulty code, or malicious spam. Wit
- \ No newline at end of file + + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-sample-application-aws-lambda.md b/develop/dev-guide-sample-application-aws-lambda.md index be63b57564aa0..2b9594de7748d 100644 --- a/develop/dev-guide-sample-application-aws-lambda.md +++ b/develop/dev-guide-sample-application-aws-lambda.md @@ -16,7 +16,7 @@ In this tutorial, you can learn how to use TiDB and mysql2 in AWS Lambda Functio > **Note** > -> This tutorial works with TiDB Serverless and TiDB Self-Hosted. +> This tutorial works with {{{ .starter }}}, {{{ .essential }}}, and TiDB Self-Managed. ## Prerequisites @@ -33,7 +33,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -41,7 +41,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -79,7 +79,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -87,7 +87,8 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` + - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -95,11 +96,11 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > > In Node.js applications, you don't have to provide an SSL CA certificate, because Node.js uses the built-in [Mozilla CA certificate](https://wiki.mozilla.org/CA/Included_Certificates) by default when establishing the TLS (SSL) connection. -4. Click **Create password** to create a random password. +4. Click **Generate Password** to create a random password. > **Tip** > - > If you have generated a password before, you can either use the original password or click **Reset password** to generate a new one. + > If you have generated a password before, you can either use the original password or click **Reset Password** to generate a new one. 5. Copy and paste the corresponding connection string into `env.json`. The following is an example: @@ -118,7 +119,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele
-
+
Copy and paste the corresponding connection string into `env.json`. The following is an example: @@ -211,20 +212,20 @@ You can deploy the AWS Lambda Function using either the [SAM CLI](#sam-cli-deplo # Setting default arguments for 'sam deploy' # ========================================= # Stack Name [sam-app]: tidb-aws-lambda-quickstart - # AWS Region [us-east-1]: + # AWS Region [us-east-1]: # #Shows you resources changes to be deployed and require a 'Y' to initiate deploy - # Confirm changes before deploy [y/N]: + # Confirm changes before deploy [y/N]: # #SAM needs permission to be able to create roles to connect to the resources in your template - # Allow SAM CLI IAM role creation [Y/n]: + # Allow SAM CLI IAM role creation [Y/n]: # #Preserves the state of previously provisioned resources when an operation fails - # Disable rollback [y/N]: + # Disable rollback [y/N]: # tidbHelloWorldFunction may not have authorization defined, Is this okay? [y/N]: y # tidbHelloWorldFunction may not have authorization defined, Is this okay? [y/N]: y # tidbHelloWorldFunction may not have authorization defined, Is this okay? [y/N]: y # tidbHelloWorldFunction may not have authorization defined, Is this okay? [y/N]: y - # Save arguments to configuration file [Y/n]: - # SAM configuration file [samconfig.toml]: - # SAM configuration environment [default]: + # Save arguments to configuration file [Y/n]: + # SAM configuration file [samconfig.toml]: + # SAM configuration environment [default]: # Looking for resources needed for deployment: # Creating the required resources... @@ -352,18 +353,28 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). - Using [connection pools](https://github.com/sidorares/node-mysql2#using-connection-pools) to manage database connections can reduce the performance overhead caused by frequently establishing and destroying connections. - To avoid SQL injection, it is recommended to use [prepared statements](https://github.com/sidorares/node-mysql2#using-prepared-statements). -- In scenarios where there are not many complex SQL statements involved, using ORM frameworks like [Sequelize](https://sequelize.org/), [TypeORM](https://typeorm.io/), or [Prisma](https://www.prisma.io/) can greatly improve development efficiency. +- In scenarios where there are not many complex SQL statements involved, using ORM frameworks like [Sequelize](https://sequelize.org/), [TypeORM](https://typeorm.io/), or [Prisma](https://www.prisma.io/) can greatly improve development efficiency. - For building a RESTful API for your application, it is recommended to [use AWS Lambda with API Gateway](https://docs.aws.amazon.com/lambda/latest/dg/services-apigateway.html). -- For designing high-performance applications using TiDB Serverless and AWS Lambda, refer to [this blog](https://aws.amazon.com/blogs/apn/designing-high-performance-applications-using-serverless-tidb-cloud-and-aws-lambda/). +- For designing high-performance applications using {{{ .starter }}} and AWS Lambda, refer to [this blog](https://aws.amazon.com/blogs/apn/designing-high-performance-applications-using-serverless-tidb-cloud-and-aws-lambda/). ## Next steps - For more details on how to use TiDB in AWS Lambda Function, see our [TiDB-Lambda-integration/aws-lambda-bookstore Demo](https://github.com/pingcap/TiDB-Lambda-integration/blob/main/aws-lambda-bookstore/README.md). You can also use AWS API Gateway to build a RESTful API for your application. -- Learn more usage of `mysql2` from [the documentation of `mysql2`](https://github.com/sidorares/node-mysql2/tree/master/documentation/en). +- Learn more usage of `mysql2` from [the documentation of `mysql2`](https://sidorares.github.io/node-mysql2/docs/documentation). - Learn more usage of AWS Lambda from [the AWS developer guide of `Lambda`](https://docs.aws.amazon.com/lambda/latest/dg/welcome.html). - Learn the best practices for TiDB application development with the chapters in the [Developer guide](/develop/dev-guide-overview.md), such as [Insert data](/develop/dev-guide-insert-data.md), [Update data](/develop/dev-guide-update-data.md), [Delete data](/develop/dev-guide-delete-data.md), [Single table reading](/develop/dev-guide-get-data-from-single-table.md), [Transactions](/develop/dev-guide-transaction-overview.md), and [SQL performance optimization](/develop/dev-guide-optimize-sql-overview.md). - Learn through the professional [TiDB developer courses](https://www.pingcap.com/education/) and earn [TiDB certifications](https://www.pingcap.com/education/certification/) after passing the exam. ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-sample-application-golang-gorm.md b/develop/dev-guide-sample-application-golang-gorm.md index fc9f6790853dc..eab223e110e10 100644 --- a/develop/dev-guide-sample-application-golang-gorm.md +++ b/develop/dev-guide-sample-application-golang-gorm.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and GORM to accomplish the follo > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -29,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -37,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -60,7 +60,7 @@ cd tidb-golang-gorm-quickstart Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -68,7 +68,8 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` + - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -76,11 +77,11 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. -4. Click **Create password** to create a random password. +4. Click **Generate Password** to create a random password. > **Tip:** > - > If you have created a password before, you can either use the original password or click **Reset password** to generate a new one. + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. 5. Run the following command to copy `.env.example` and rename it to `.env`: @@ -101,20 +102,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele Be sure to replace the placeholders `{}` with the connection parameters obtained from the connection dialog. - TiDB Serverless requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. + {{{ .starter }}} requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -138,7 +141,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -198,7 +201,7 @@ func createDB() *gorm.DB { } ``` -When using this function, you need to replace `${tidb_host}`, `${tidb_port}`, `${tidb_user}`, `${tidb_password}`, and `${tidb_db_name}` with the actual values of your TiDB cluster. TiDB Serverless requires a secure connection. Therefore, you need to set the value of `${use_ssl}` to `true`. +When using this function, you need to replace `${tidb_host}`, `${tidb_port}`, `${tidb_user}`, `${tidb_password}`, and `${tidb_db_name}` with the actual values of your TiDB cluster. {{{ .starter }}} requires a secure connection. Therefore, you need to set the value of `${use_ssl}` to `true`. ### Insert data @@ -241,4 +244,14 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-sample-application-golang-sql-driver.md b/develop/dev-guide-sample-application-golang-sql-driver.md index f88df0c5e40eb..01763940cae6e 100644 --- a/develop/dev-guide-sample-application-golang-sql-driver.md +++ b/develop/dev-guide-sample-application-golang-sql-driver.md @@ -1,7 +1,6 @@ --- title: Connect to TiDB with Go-MySQL-Driver summary: Learn how to connect to TiDB using Go-MySQL-Driver. This tutorial gives Golang sample code snippets that work with TiDB using Go-MySQL-Driver. -aliases: ['/tidb/dev/dev-guide-outdated-for-go-sql-driver-mysql','/tidb/dev/dev-guide-outdated-for-gorm','/tidb/dev/dev-guide-sample-application-golang'] --- # Connect to TiDB with Go-MySQL-Driver @@ -16,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and Go-MySQL-Driver to accomplis > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -30,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -38,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -61,7 +60,7 @@ cd tidb-golang-sql-driver-quickstart Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -69,7 +68,8 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` + - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -77,11 +77,11 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. -4. Click **Create password** to create a random password. +4. Click **Generate Password** to create a random password. > **Tip:** > - > If you have created a password before, you can either use the original password or click **Reset password** to generate a new one. + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. 5. Run the following command to copy `.env.example` and rename it to `.env`: @@ -102,20 +102,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele Be sure to replace the placeholders `{}` with the connection parameters obtained from the connection dialog. - TiDB Serverless requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. + {{{ .starter }}} requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -139,7 +141,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -197,7 +199,7 @@ func openDB(driverName string, runnable func(db *sql.DB)) { } ``` -When using this function, you need to replace `${tidb_host}`, `${tidb_port}`, `${tidb_user}`, `${tidb_password}`, and `${tidb_db_name}` with the actual values of your TiDB cluster. TiDB Serverless requires a secure connection. Therefore, you need to set the value of `${use_ssl}` to `true`. +When using this function, you need to replace `${tidb_host}`, `${tidb_port}`, `${tidb_user}`, `${tidb_password}`, and `${tidb_db_name}` with the actual values of your TiDB cluster. {{{ .starter }}} and {{{ .essential }}} require a secure connection. Therefore, you need to set the value of `${use_ssl}` to `true`. ### Insert data @@ -292,4 +294,14 @@ Unless you need to write complex SQL statements, it is recommended to use [ORM]( ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-sample-application-java-hibernate.md b/develop/dev-guide-sample-application-java-hibernate.md index 427dd79f68641..de721bee9e4f1 100644 --- a/develop/dev-guide-sample-application-java-hibernate.md +++ b/develop/dev-guide-sample-application-java-hibernate.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and Hibernate to accomplish the > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -30,7 +30,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -38,7 +38,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -61,7 +61,7 @@ cd tidb-java-hibernate-quickstart Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -69,7 +69,8 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` + - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -77,11 +78,11 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. -4. Click **Create password** to create a random password. +4. Click **Generate Password** to create a random password. > **Tip:** > - > If you have created a password before, you can either use the original password or click **Reset password** to generate a new one. + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. 5. Run the following command to copy `env.sh.example` and rename it to `env.sh`: @@ -102,20 +103,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele Be sure to replace the placeholders `{}` with the connection parameters obtained from the connection dialog. - TiDB Serverless requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. + {{{ .starter }}} requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. 7. Save the `env.sh` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `env.sh.example` and rename it to `env.sh`: @@ -139,7 +142,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `env.sh` file.
-
+
1. Run the following command to copy `env.sh.example` and rename it to `env.sh`: @@ -264,4 +267,14 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-sample-application-java-jdbc.md b/develop/dev-guide-sample-application-java-jdbc.md index d3753c652b415..5b5d085f521ef 100644 --- a/develop/dev-guide-sample-application-java-jdbc.md +++ b/develop/dev-guide-sample-application-java-jdbc.md @@ -1,6 +1,7 @@ --- title: Connect to TiDB with JDBC summary: Learn how to connect to TiDB using JDBC. This tutorial gives Java sample code snippets that work with TiDB using JDBC. +aliases: ['/tidb/v7.5/dev-guide-sample-application-java'] --- # Connect to TiDB with JDBC @@ -13,9 +14,23 @@ In this tutorial, you can learn how to use TiDB and JDBC to accomplish the follo - Connect to your TiDB cluster using JDBC. - Build and run your application. Optionally, you can find [sample code snippets](#sample-code-snippets) for basic CRUD operations. + + +> **Note:** +> +> - This tutorial works with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed. +> - Starting from TiDB v7.4, if `connectionCollation` is not configured, and `characterEncoding` is either not configured or set to `UTF-8` in the JDBC URL, the collation used in a JDBC connection depends on the JDBC driver version. For more information, see [Collation used in JDBC connections](/faq/sql-faq.md#collation-used-in-jdbc-connections). + + + + + > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> - This tutorial works with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed. +> - Starting from TiDB v7.4, if `connectionCollation` is not configured, and `characterEncoding` is either not configured or set to `UTF-8` in the JDBC URL, the collation used in a JDBC connection depends on the JDBC driver version. For more information, see [Collation used in JDBC connections](https://docs.pingcap.com/tidb/stable/sql-faq#collation-used-in-jdbc-connections). + + ## Prerequisites @@ -30,15 +45,19 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. +> **Note:** +> +> For security considerations, it is recommended that you use `VERIFY_IDENTITY` to establish TLS connections to TiDB clusters when connecting over the internet. {{{ .starter }}}, {{{ .essential }}}, and TiDB Cloud Dedicated use Subject Alternative Name (SAN) certificates, which require MySQL Connector/J version to be greater than or equal to [8.0.22](https://dev.mysql.com/doc/relnotes/connector-j/en/news-8-0-22.html). + **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -61,7 +80,7 @@ cd tidb-java-jdbc-quickstart Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -69,7 +88,8 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` + - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -77,11 +97,11 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. -4. Click **Create password** to create a random password. +4. Click **Generate Password** to create a random password. > **Tip:** > - > If you have created a password before, you can either use the original password or click **Reset password** to generate a new one. + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. 5. Run the following command to copy `env.sh.example` and rename it to `env.sh`: @@ -102,20 +122,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele Be sure to replace the placeholders `{}` with the connection parameters obtained from the connection dialog. - TiDB Serverless requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. + {{{ .starter }}} requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. 7. Save the `env.sh` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. + + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `env.sh.example` and rename it to `env.sh`: @@ -139,7 +161,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `env.sh` file.
-
+
1. Run the following command to copy `env.sh.example` and rename it to `env.sh`: @@ -291,11 +313,21 @@ Unless you need to write complex SQL statements, it is recommended to use [ORM]( ## Next steps -- Learn more usage of MySQL Connector/J from [the documentation of MySQL Connector/J](https://dev.mysql.com/doc/connector-j/8.1/en/). +- Learn more usage of MySQL Connector/J from [the documentation of MySQL Connector/J](https://dev.mysql.com/doc/connector-j/en/). - Learn the best practices for TiDB application development with the chapters in the [Developer guide](/develop/dev-guide-overview.md), such as [Insert data](/develop/dev-guide-insert-data.md), [Update data](/develop/dev-guide-update-data.md), [Delete data](/develop/dev-guide-delete-data.md), [Single table reading](/develop/dev-guide-get-data-from-single-table.md), [Transactions](/develop/dev-guide-transaction-overview.md), and [SQL performance optimization](/develop/dev-guide-optimize-sql-overview.md). - Learn through the professional [TiDB developer courses](https://www.pingcap.com/education/) and earn [TiDB certifications](https://www.pingcap.com/education/certification/) after passing the exam. - Learn through the course for Java developers: [Working with TiDB from Java](https://eng.edu.pingcap.com/catalog/info/id:212). ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-sample-application-java-mybatis.md b/develop/dev-guide-sample-application-java-mybatis.md index 42f06f386681e..e778d725e7cdd 100644 --- a/develop/dev-guide-sample-application-java-mybatis.md +++ b/develop/dev-guide-sample-application-java-mybatis.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and MyBatis to accomplish the fo > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -30,7 +30,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -38,7 +38,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -61,7 +61,7 @@ cd tidb-java-mybatis-quickstart Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -69,7 +69,8 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` + - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -77,11 +78,11 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. -4. Click **Create password** to create a random password. +4. Click **Generate Password** to create a random password. > **Tip:** > - > If you have created a password before, you can either use the original password or click **Reset password** to generate a new one. + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. 5. Run the following command to copy `env.sh.example` and rename it to `env.sh`: @@ -102,20 +103,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele Be sure to replace the placeholders `{}` with the connection parameters obtained from the connection dialog. - TiDB Serverless requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. + {{{ .starter }}} requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. 7. Save the `env.sh` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `env.sh.example` and rename it to `env.sh`: @@ -139,7 +142,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `env.sh` file.
-
+
1. Run the following command to copy `env.sh.example` and rename it to `env.sh`: @@ -314,4 +317,14 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-sample-application-java-spring-boot.md b/develop/dev-guide-sample-application-java-spring-boot.md index 9e224a708fb10..b42dda4d83584 100644 --- a/develop/dev-guide-sample-application-java-spring-boot.md +++ b/develop/dev-guide-sample-application-java-spring-boot.md @@ -1,7 +1,6 @@ --- title: Connect to TiDB with Spring Boot summary: Learn how to connect to TiDB using Spring Boot. This tutorial gives Java sample code snippets that work with TiDB using Spring Boot. -aliases: ['/tidbcloud/dev-guide-sample-application-spring-boot','/tidb/dev/dev-guide-sample-application-spring-boot'] --- # Connect to TiDB with Spring Boot @@ -16,7 +15,7 @@ In this tutorial, you can learn how to use TiDB along with [Spring Data JPA](htt > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -31,7 +30,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -39,7 +38,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -62,7 +61,7 @@ cd tidb-java-springboot-jpa-quickstart Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -70,7 +69,8 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` + - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -78,11 +78,11 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. -4. Click **Create password** to create a random password. +4. Click **Generate Password** to create a random password. > **Tip:** > - > If you have created a password before, you can either use the original password or click **Reset password** to generate a new one. + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. 5. Run the following command to copy `env.sh.example` and rename it to `env.sh`: @@ -103,20 +103,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele Be sure to replace the placeholders `{}` with the connection parameters obtained from the connection dialog. - TiDB Serverless requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. + {{{ .starter }}} requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. 7. Save the `env.sh` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `env.sh.example` and rename it to `env.sh`: @@ -140,7 +142,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `env.sh` file.
-
+
1. Run the following command to copy `env.sh.example` and rename it to `env.sh`: @@ -255,7 +257,6 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). ## Next steps -- Learn more usage of Hibernate from [the documentation of Hibernate](https://hibernate.org/orm/documentation). - Learn more usage about the third-party libraries and frameworks used in this document, refer to their official documentation: - [The documentation of Spring Framework](https://spring.io/projects/spring-framework) @@ -269,4 +270,14 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-sample-application-nextjs.md b/develop/dev-guide-sample-application-nextjs.md index fa73096b00145..bb1a50374f183 100644 --- a/develop/dev-guide-sample-application-nextjs.md +++ b/develop/dev-guide-sample-application-nextjs.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and mysql2 in Next.js to accompl > **Note** > -> This tutorial works with TiDB Serverless and TiDB Self-Hosted. +> This tutorial works with {{{ .starter }}}, {{{ .essential }}}, and TiDB Self-Managed. ## Prerequisites @@ -23,13 +23,13 @@ To complete this tutorial, you need: - [Node.js **18**](https://nodejs.org/en/download/) or later. - [Git](https://git-scm.com/downloads). -- A TiDB cluster. +- A TiDB cluster. **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -37,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -73,7 +73,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele -
+
1. Navigate to the [**Clusters** page](https://tidbcloud.com/console/clusters), and then click the name of your target cluster to go to its overview page. @@ -81,7 +81,8 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` + - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -89,11 +90,11 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > > In Node.js applications, you do not have to provide an SSL CA certificate, because Node.js uses the built-in [Mozilla CA certificate](https://wiki.mozilla.org/CA/Included_Certificates) by default when establishing the TLS (SSL) connection. -4. Click **Create password** to create a random password. +4. Click **Generate Password** to create a random password. > **Tip** > - > If you have created a password before, you can either use the original password or click **Reset password** to generate a new one. + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. 5. Run the following command to copy `.env.example` and rename it to `.env`: @@ -123,7 +124,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -276,10 +277,20 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). ## Next steps - For more details on how to build a complex application with ORM and Next.js, see [our Bookshop Demo](https://github.com/pingcap/tidb-prisma-vercel-demo). -- Learn more usage of node-mysql2 driver from [the documentation of node-mysql2](https://github.com/sidorares/node-mysql2/tree/master/documentation/en). +- Learn more usage of node-mysql2 driver from [the documentation of node-mysql2](https://sidorares.github.io/node-mysql2/docs/documentation). - Learn the best practices for TiDB application development with the chapters in the [Developer guide](/develop/dev-guide-overview.md), such as [Insert data](/develop/dev-guide-insert-data.md), [Update data](/develop/dev-guide-update-data.md), [Delete data](/develop/dev-guide-delete-data.md), [Single table reading](/develop/dev-guide-get-data-from-single-table.md), [Transactions](/develop/dev-guide-transaction-overview.md), and [SQL performance optimization](/develop/dev-guide-optimize-sql-overview.md). - Learn through the professional [TiDB developer courses](https://www.pingcap.com/education/) and earn [TiDB certifications](https://www.pingcap.com/education/certification/) after passing the exam. ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-sample-application-nodejs-mysql2.md b/develop/dev-guide-sample-application-nodejs-mysql2.md index c2e2c9cd9f656..1124fb1ed778c 100644 --- a/develop/dev-guide-sample-application-nodejs-mysql2.md +++ b/develop/dev-guide-sample-application-nodejs-mysql2.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and node-mysql2 to accomplish th > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -29,13 +29,13 @@ To complete this tutorial, you need: -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -77,7 +77,7 @@ npm install mysql2 dotenv --save Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -85,11 +85,12 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public`. + - **Connection Type** is set to `Public`. + - **Branch** is set to `main`. - **Connect With** is set to `General`. - **Operating System** matches the operating system where you run the application. -4. If you have not set a password yet, click **Create password** to generate a random password. +4. If you have not set a password yet, click **Generate Password** to generate a random password. 5. Run the following command to copy `.env.example` and rename it to `.env`: @@ -110,20 +111,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > **Note** > - > For TiDB Serverless, TLS connection **MUST** be enabled via `TIDB_ENABLE_SSL` when using public endpoint. + > For {{{ .starter }}}, TLS connection **MUST** be enabled via `TIDB_ENABLE_SSL` when using public endpoint. 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -145,14 +148,14 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > **Note** > - > It is recommended to enable TLS connection when using the public endpoint to connect to TiDB Dedicated. + > It is recommended to enable TLS connection when using the public endpoint to connect to TiDB Cloud Dedicated. > > To enable TLS connection, modify `TIDB_ENABLE_SSL` to `true` and using `TIDB_CA_PATH` to specify the file path of CA certificate downloaded from the connection dialog. 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -188,7 +191,7 @@ npm start If the connection is successful, the console will output the version of the TiDB cluster as follows: ``` -🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v7.4.0) +🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v{{{ .tidb-version }}}) ⏳ Loading sample game data... ✅ Loaded sample game data. @@ -243,7 +246,7 @@ void main(); > **Note** > -> For TiDB Serverless, you **MUST** enable TLS connection via `TIDB_ENABLE_SSL` when using public endpoint. However, you **don't** have to specify an SSL CA certificate via `TIDB_CA_PATH`, because Node.js uses the built-in [Mozilla CA certificate](https://wiki.mozilla.org/CA/Included_Certificates) by default, which is trusted by TiDB Serverless. +> For {{{ .starter }}} and {{{ .essential }}}, you **MUST** enable TLS connection via `TIDB_ENABLE_SSL` when using public endpoint. However, you **don't** have to specify an SSL CA certificate via `TIDB_CA_PATH`, because Node.js uses the built-in [Mozilla CA certificate](https://wiki.mozilla.org/CA/Included_Certificates) by default, which is trusted by {{{ .starter }}}. ### Insert data @@ -308,4 +311,14 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). \ No newline at end of file + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-sample-application-nodejs-mysqljs.md b/develop/dev-guide-sample-application-nodejs-mysqljs.md index e2c1a38bc35cb..5799e361dd2fa 100644 --- a/develop/dev-guide-sample-application-nodejs-mysqljs.md +++ b/develop/dev-guide-sample-application-nodejs-mysqljs.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and mysql.js driver to accomplis > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -29,13 +29,13 @@ To complete this tutorial, you need: -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -77,7 +77,7 @@ npm install mysql dotenv --save Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -85,11 +85,12 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public`. + - **Connection Type** is set to `Public`. + - **Branch** is set to `main`. - **Connect With** is set to `General`. - **Operating System** matches the operating system where you run the application. -4. If you have not set a password yet, click **Create password** to generate a random password. +4. If you have not set a password yet, click **Generate Password** to generate a random password. 5. Run the following command to copy `.env.example` and rename it to `.env`: @@ -110,20 +111,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > **Note** > - > For TiDB Serverless, TLS connection **MUST** be enabled via `TIDB_ENABLE_SSL` when using public endpoint. + > For {{{ .starter }}}, TLS connection **MUST** be enabled via `TIDB_ENABLE_SSL` when using public endpoint. 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -145,14 +148,14 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > **Note** > - > It is recommended to enable TLS connection when using the public endpoint to connect to TiDB Dedicated. + > It is recommended to enable TLS connection when using the public endpoint to connect to TiDB Cloud Dedicated. > > To enable TLS connection, modify `TIDB_ENABLE_SSL` to `true` and using `TIDB_CA_PATH` to specify the file path of CA certificate downloaded from the connection dialog. 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -188,7 +191,7 @@ npm start If the connection is successful, the console will output the version of the TiDB cluster as follows: ``` -🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v7.4.0) +🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v{{{ .tidb-version }}}) ⏳ Loading sample game data... ✅ Loaded sample game data. @@ -239,7 +242,7 @@ conn.end(); > **Note** > -> For TiDB Serverless, you **MUST** enable TLS connection via `TIDB_ENABLE_SSL` when using public endpoint. However, you **don't** have to specify an SSL CA certificate via `TIDB_CA_PATH`, because Node.js uses the built-in [Mozilla CA certificate](https://wiki.mozilla.org/CA/Included_Certificates) by default, which is trusted by TiDB Serverless. +> For {{{ .starter }}} and {{{ .essential }}}, you **MUST** enable TLS connection via `TIDB_ENABLE_SSL` when using public endpoint. However, you **don't** have to specify an SSL CA certificate via `TIDB_CA_PATH`, because Node.js uses the built-in [Mozilla CA certificate](https://wiki.mozilla.org/CA/Included_Certificates) by default, which is trusted by {{{ .starter }}}. ### Insert data @@ -331,4 +334,14 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-sample-application-nodejs-prisma.md b/develop/dev-guide-sample-application-nodejs-prisma.md index ac1484546223c..d93a0c8d97083 100644 --- a/develop/dev-guide-sample-application-nodejs-prisma.md +++ b/develop/dev-guide-sample-application-nodejs-prisma.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and Prisma to accomplish the fol > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -29,13 +29,13 @@ To complete this tutorial, you need: -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -77,7 +77,7 @@ npm install prisma typescript ts-node @types/node --save-dev Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -85,11 +85,12 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public`. - - **Connect With** is set to `General`. + - **Connection Type** is set to `Public`. + - **Branch** is set to `main`. + - **Connect With** is set to `Prisma`. - **Operating System** matches the operating system where you run the application. -4. If you have not set a password yet, click **Create password** to generate a random password. +4. If you have not set a password yet, click **Generate Password** to generate a random password. 5. Run the following command to copy `.env.example` and rename it to `.env`: @@ -97,15 +98,15 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele cp .env.example .env ``` -6. Edit the `.env` file, set up the environment variable `DATABASE_URL` as follows, replace the corresponding placeholders `{}` with connection parameters on the connection dialog: +6. Edit the `.env` file, set up the environment variable `DATABASE_URL` as follows, and replace the corresponding placeholders `{}` with the connection string in the connection dialog: ```dotenv - DATABASE_URL=mysql://{user}:{password}@{host}:4000/test?sslaccept=strict + DATABASE_URL='{connection_string}' ``` > **Note** > - > For TiDB Serverless, you **MUST** enable TLS connection by setting `sslaccept=strict` when using public endpoint. + > For {{{ .starter }}}, you **MUST** enable TLS connection by setting `sslaccept=strict` when using public endpoint. 7. Save the `.env` file. 8. In the `prisma/schema.prisma`, set up `mysql` as the connection provider and `env("DATABASE_URL")` as the connection URL: @@ -118,15 +119,17 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele ```
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -137,12 +140,12 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 5. Edit the `.env` file, set up the environment variable `DATABASE_URL` as follows, replace the corresponding placeholders `{}` with connection parameters on the connection dialog: ```dotenv - DATABASE_URL=mysql://{user}:{password}@{host}:4000/test?sslaccept=strict&sslcert={downloaded_ssl_ca_path} + DATABASE_URL='mysql://{user}:{password}@{host}:4000/test?sslaccept=strict&sslcert={downloaded_ssl_ca_path}' ``` > **Note** > - > For TiDB Serverless, It is **RECOMMENDED** to enable TLS connection by setting `sslaccept=strict` when using public endpoint. When you set up `sslaccept=strict` to enable TLS connection, you **MUST** specify the file path of the CA certificate downloaded from connection dialog via `sslcert=/path/to/ca.pem`. + > For {{{ .starter }}}, It is **RECOMMENDED** to enable TLS connection by setting `sslaccept=strict` when using public endpoint. When you set up `sslaccept=strict` to enable TLS connection, you **MUST** specify the file path of the CA certificate downloaded from connection dialog via `sslcert=/path/to/ca.pem`. 6. Save the `.env` file. 7. In the `prisma/schema.prisma`, set up `mysql` as the connection provider and `env("DATABASE_URL")` as the connection URL: @@ -155,7 +158,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele ```
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -166,7 +169,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Edit the `.env` file, set up the environment variable `DATABASE_URL` as follows, replace the corresponding placeholders `{}` with connection parameters of your TiDB cluster: ```dotenv - DATABASE_URL=mysql://{user}:{password}@{host}:4000/test + DATABASE_URL='mysql://{user}:{password}@{host}:4000/test' ``` If you are running TiDB locally, the default host address is `127.0.0.1`, and the password is empty. @@ -267,7 +270,7 @@ void main(); If the connection is successful, the terminal will output the version of the TiDB cluster as follows: ``` -🔌 Connected to TiDB cluster! (TiDB version: 5.7.25-TiDB-v6.6.0-serverless) +🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v{{{ .tidb-version }}}) 🆕 Created a new player with ID 1. ℹ️ Got Player 1: Player { id: 1, coins: 100, goods: 100 } 🔢 Added 50 coins and 50 goods to player 1, now player 1 has 150 coins and 150 goods. @@ -369,4 +372,14 @@ To check [referential integrity](https://en.wikipedia.org/wiki/Referential_integ ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-sample-application-nodejs-sequelize.md b/develop/dev-guide-sample-application-nodejs-sequelize.md index c4c905fe75185..9c0e718308cef 100644 --- a/develop/dev-guide-sample-application-nodejs-sequelize.md +++ b/develop/dev-guide-sample-application-nodejs-sequelize.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and Sequelize to accomplish the > **Note** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -29,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -37,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -73,7 +73,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -81,7 +81,8 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` + - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -89,11 +90,11 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > > In Node.js applications, you don't have to provide an SSL CA certificate, because Node.js uses the built-in [Mozilla CA certificate](https://wiki.mozilla.org/CA/Included_Certificates) by default when establishing the TLS (SSL) connection. -4. Click **Create password** to create a random password. +4. Click **Generate Password** to create a random password. > **Tip** > - > If you have generated a password before, you can either use the original password or click **Reset password** to generate a new one. + > If you have generated a password before, you can either use the original password or click **Reset Password** to generate a new one. 5. Run the following command to copy `.env.example` and rename it to `.env`: @@ -102,7 +103,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele ``` 6. Edit the `.env` file, set up the environment variables as follows, replace the corresponding placeholders `{}` with connection parameters on the connection dialog: - + ```dotenv TIDB_HOST='{host}' TIDB_PORT='4000' @@ -116,15 +117,17 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. + + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -148,7 +151,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -322,4 +325,14 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). ## Need help? -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-sample-application-nodejs-typeorm.md b/develop/dev-guide-sample-application-nodejs-typeorm.md index 5f3190e9ca552..0948a9598d077 100644 --- a/develop/dev-guide-sample-application-nodejs-typeorm.md +++ b/develop/dev-guide-sample-application-nodejs-typeorm.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and TypeORM to accomplish the fo > **Note** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -29,13 +29,13 @@ To complete this tutorial, you need: -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -85,7 +85,7 @@ npm install @types/node ts-node typescript --save-dev Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -93,11 +93,12 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public`. + - **Connection Type** is set to `Public`. + - **Branch** is set to `main`. - **Connect With** is set to `General`. - **Operating System** matches the operating system where you run the application. -4. If you have not set a password yet, click **Create password** to generate a random password. +4. If you have not set a password yet, click **Generate Password** to generate a random password. 5. Run the following command to copy `.env.example` and rename it to `.env`: @@ -118,20 +119,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > **Note** > - > For TiDB Serverless, you **MUST** enable TLS connection via `TIDB_ENABLE_SSL` when using public endpoint. + > For {{{ .starter }}} and {{{ .essential }}}, you **MUST** enable TLS connection via `TIDB_ENABLE_SSL` when using public endpoint. 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -153,12 +156,12 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > **Note** > - > For TiDB Dedicated, it is **RECOMMENDED** to enable TLS connection via `TIDB_ENABLE_SSL` when using public endpoint. When you set up `TIDB_ENABLE_SSL=true`, you **MUST** specify the path of the CA certificate downloaded from connection dialog via `TIDB_CA_PATH=/path/to/ca.pem`. + > For TiDB Cloud Dedicated, it is **RECOMMENDED** to enable TLS connection via `TIDB_ENABLE_SSL` when using public endpoint. When you set up `TIDB_ENABLE_SSL=true`, you **MUST** specify the path of the CA certificate downloaded from connection dialog via `TIDB_CA_PATH=/path/to/ca.pem`. 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -230,7 +233,7 @@ npm start If the connection is successful, the terminal will output the version of the TiDB cluster as follows: ``` -🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v7.4.0) +🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v{{{ .tidb-version }}}) 🆕 Created a new player with ID 2. ℹ️ Got Player 2: Player { id: 2, coins: 100, goods: 100 } 🔢 Added 50 coins and 50 goods to player 2, now player 2 has 100 coins and 150 goods. @@ -273,9 +276,9 @@ export const AppDataSource = new DataSource({ > **Note** > -> For TiDB Serverless, you MUST enable TLS connection when using public endpoint. In this sample code, please set up the environment variable `TIDB_ENABLE_SSL` in the `.env` file to `true`. +> For {{{ .starter }}} and {{{ .essential }}}, you MUST enable TLS connection when using public endpoint. In this sample code, please set up the environment variable `TIDB_ENABLE_SSL` in the `.env` file to `true`. > -> However, you **don't** have to specify an SSL CA certificate via `TIDB_CA_PATH`, because Node.js uses the built-in [Mozilla CA certificate](https://wiki.mozilla.org/CA/Included_Certificates) by default, which is trusted by TiDB Serverless. +> However, you **don't** have to specify an SSL CA certificate via `TIDB_CA_PATH`, because Node.js uses the built-in [Mozilla CA certificate](https://wiki.mozilla.org/CA/Included_Certificates) by default, which is trusted by {{{ .starter }}} and {{{ .essential }}}. ### Insert data @@ -368,4 +371,14 @@ For more information, refer to the [TypeORM FAQ](https://typeorm.io/relations-fa ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-sample-application-python-django.md b/develop/dev-guide-sample-application-python-django.md index 1c8e00dbfee8b..18f33d9f50105 100644 --- a/develop/dev-guide-sample-application-python-django.md +++ b/develop/dev-guide-sample-application-python-django.md @@ -1,7 +1,6 @@ --- title: Connect to TiDB with Django summary: Learn how to connect to TiDB using Django. This tutorial gives Python sample code snippets that work with TiDB using Django. -aliases: ['/tidb/dev/dev-guide-outdated-for-django'] --- # Connect to TiDB with Django @@ -16,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and Django to accomplish the fol > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted clusters. +> This tutorial works with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed clusters. ## Prerequisites @@ -30,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -38,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -79,7 +78,7 @@ For more information, refer to [django-tidb repository](https://github.com/pingc Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -87,7 +86,8 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` + - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -95,11 +95,11 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. -4. Click **Create password** to create a random password. +4. Click **Generate Password** to create a random password. > **Tip:** > - > If you have created a password before, you can either use the original password or click **Reset password** to generate a new one. + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. 5. Run the following command to copy `.env.example` and rename it to `.env`: @@ -120,20 +120,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele Be sure to replace the placeholders `{}` with the connection parameters obtained from the connection dialog. - TiDB Serverless requires a secure connection. Since the `ssl_mode` of mysqlclient defaults to `PREFERRED`, you don't need to manually specify `CA_PATH`. Just leave it empty. But if you have a special reason to specify `CA_PATH` manually, you can refer to the [TLS connections to TiDB Serverless](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters) to get the certificate paths for different operating systems. + {{{ .starter }}} requires a secure connection. Since the `ssl_mode` of mysqlclient defaults to `PREFERRED`, you don't need to manually specify `CA_PATH`. Just leave it empty. But if you have a special reason to specify `CA_PATH` manually, you can refer to the [TLS connections to {{{ .starter }}}](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters) to get the certificate paths for different operating systems. 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -157,7 +159,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -329,4 +331,14 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-sample-application-python-mysql-connector.md b/develop/dev-guide-sample-application-python-mysql-connector.md index a6f27f3839d97..7c5c3bc38d15c 100644 --- a/develop/dev-guide-sample-application-python-mysql-connector.md +++ b/develop/dev-guide-sample-application-python-mysql-connector.md @@ -1,7 +1,6 @@ --- title: Connect to TiDB with MySQL Connector/Python summary: Learn how to connect to TiDB using MySQL Connector/Python. This tutorial gives Python sample code snippets that work with TiDB using MySQL Connector/Python. -aliases: ['/tidb/dev/dev-guide-sample-application-python','/tidb/dev/dev-guide-outdated-for-python-mysql-connector'] --- # Connect to TiDB with MySQL Connector/Python @@ -16,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and MySQL Connector/Python to ac > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted clusters. +> This tutorial works with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed clusters. ## Prerequisites @@ -30,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -38,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -69,7 +68,7 @@ pip install -r requirements.txt Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -77,7 +76,8 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` + - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -85,11 +85,11 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. -4. Click **Create password** to create a random password. +4. Click **Generate Password** to create a random password. > **Tip:** > - > If you have created a password before, you can either use the original password or click **Reset password** to generate a new one. + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. 5. Run the following command to copy `.env.example` and rename it to `.env`: @@ -113,15 +113,17 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -145,7 +147,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -280,4 +282,14 @@ Unless you need to write complex SQL statements, it is recommended to use [ORM]( ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-sample-application-python-mysqlclient.md b/develop/dev-guide-sample-application-python-mysqlclient.md index df0d202d38455..03e59b59db5d4 100644 --- a/develop/dev-guide-sample-application-python-mysqlclient.md +++ b/develop/dev-guide-sample-application-python-mysqlclient.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and mysqlclient to accomplish th > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -29,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -37,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -70,7 +70,7 @@ If you encounter installation issues, refer to the [mysqlclient official documen Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -78,7 +78,8 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` + - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -86,11 +87,11 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. -4. Click **Create password** to create a random password. +4. Click **Generate Password** to create a random password. > **Tip:** > - > If you have created a password before, you can either use the original password or click **Reset password** to generate a new one. + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. 5. Run the following command to copy `.env.example` and rename it to `.env`: @@ -111,20 +112,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele Be sure to replace the placeholders `{}` with the connection parameters obtained from the connection dialog. - TiDB Serverless requires a secure connection. Since the `ssl_mode` of mysqlclient defaults to `PREFERRED`, you don't need to manually specify `CA_PATH`. Just leave it empty. But if you have a special reason to specify `CA_PATH` manually, you can refer to the [TLS connections to TiDB Serverless](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters) to get the certificate paths for different operating systems. + {{{ .starter }}} requires a secure connection. Since the `ssl_mode` of mysqlclient defaults to `PREFERRED`, you don't need to manually specify `CA_PATH`. Just leave it empty. But if you have a special reason to specify `CA_PATH` manually, you can refer to the [TLS connections to {{{ .starter }}}](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters) to get the certificate paths for different operating systems. 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -148,7 +151,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -281,4 +284,14 @@ Unless you need to write complex SQL statements, it is recommended to use [ORM]( ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-sample-application-python-peewee.md b/develop/dev-guide-sample-application-python-peewee.md index a8183a7c17e23..03ad8210cc518 100644 --- a/develop/dev-guide-sample-application-python-peewee.md +++ b/develop/dev-guide-sample-application-python-peewee.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and peewee to accomplish the fol > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted clusters. +> This tutorial works with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed clusters. ## Prerequisites @@ -29,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -37,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -72,15 +72,16 @@ peewee is an ORM library that works with multiple databases. It provides a high- Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
-1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. +1. Navigate to the [**Clusters**](https://{{{.console-url}}}/project/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` + - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -88,11 +89,11 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. -4. Click **Create password** to create a random password. +4. Click **Generate Password** to create a random password. > **Tip:** > - > If you have created a password before, you can either use the original password or click **Reset password** to generate a new one. + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. 5. Run the following command to copy `.env.example` and rename it to `.env`: @@ -116,15 +117,17 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. + + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -148,7 +151,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -304,4 +307,14 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-sample-application-python-pymysql.md b/develop/dev-guide-sample-application-python-pymysql.md index 64fefd08cdeae..ef3b1c82fc1b3 100644 --- a/develop/dev-guide-sample-application-python-pymysql.md +++ b/develop/dev-guide-sample-application-python-pymysql.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and PyMySQL to accomplish the fo > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted clusters. +> This tutorial works with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed clusters. ## Prerequisites @@ -29,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -37,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -68,7 +68,7 @@ pip install -r requirements.txt Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -76,7 +76,8 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` + - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -84,11 +85,11 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. -4. Click **Create password** to create a random password. +4. Click **Generate Password** to create a random password. > **Tip:** > - > If you have created a password before, you can either use the original password or click **Reset password** to generate a new one. + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. 5. Run the following command to copy `.env.example` and rename it to `.env`: @@ -112,15 +113,17 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -144,7 +147,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -284,4 +287,14 @@ Unless you need to write complex SQL statements, it is recommended to use [ORM]( ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-sample-application-python-sqlalchemy.md b/develop/dev-guide-sample-application-python-sqlalchemy.md index bb3a48f9bd9aa..61327f311d899 100644 --- a/develop/dev-guide-sample-application-python-sqlalchemy.md +++ b/develop/dev-guide-sample-application-python-sqlalchemy.md @@ -1,7 +1,6 @@ --- title: Connect to TiDB with SQLAlchemy summary: Learn how to connect to TiDB using SQLAlchemy. This tutorial gives Python sample code snippets that work with TiDB using SQLAlchemy. -aliases: ['/tidb/dev/dev-guide-outdated-for-sqlalchemy'] --- # Connect to TiDB with SQLAlchemy @@ -16,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and SQLAlchemy to accomplish the > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted clusters. +> This tutorial works with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed clusters. ## Prerequisites @@ -30,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -38,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -75,7 +74,11 @@ You can also use other database drivers, such as [mysqlclient](https://github.co Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
+ +> **Note:** +> +> Currently, {{{ .starter }}} clusters have a limitation: if there are no active connections for 5 minutes, they will shut down, which closes all connections. Therefore, when using SQLAlchemy with {{{ .starter }}} clusters, pooled connections might encounter `OperationalError` such as `Lost connection to MySQL server during query` or `MySQL Connection not available`. To avoid this error, you can set the `pool_recycle` parameter to `300`. For more information, see [Dealing with Disconnects](https://docs.sqlalchemy.org/en/20/core/pooling.html#dealing-with-disconnects) in SQLAlchemy documentation. 1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -83,7 +86,8 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` + - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -91,11 +95,11 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. -4. Click **Create password** to create a random password. +4. Click **Generate Password** to create a random password. > **Tip:** > - > If you have created a password before, you can either use the original password or click **Reset password** to generate a new one. + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. 5. Run the following command to copy `.env.example` and rename it to `.env`: @@ -119,15 +123,17 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. + + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -151,7 +157,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -294,4 +300,14 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](https://support.pingcap.com/). + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-sample-application-ruby-mysql2.md b/develop/dev-guide-sample-application-ruby-mysql2.md index ddfbf9a58f61b..2e5477d7966af 100644 --- a/develop/dev-guide-sample-application-ruby-mysql2.md +++ b/develop/dev-guide-sample-application-ruby-mysql2.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and mysql2 to accomplish the fol > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -30,13 +30,13 @@ To complete this tutorial, you need: -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -78,7 +78,7 @@ bundle add mysql2 dotenv Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -86,11 +86,12 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public`. + - **Connection Type** is set to `Public`. + - **Branch** is set to `main`. - **Connect With** is set to `General`. - **Operating System** matches the operating system where you run the application. -4. If you have not set a password yet, click **Create password** to generate a random password. +4. If you have not set a password yet, click **Generate Password** to generate a random password. 5. Run the following command to copy `.env.example` and rename it to `.env`: @@ -98,33 +99,35 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele cp .env.example .env ``` -6. Edit the `.env` file, set up the environment variables as follows, and replace the corresponding placeholders `<>` with connection parameters in the connection dialog: +6. Edit the `.env` file, set up the environment variables as follows, and replace the corresponding placeholders `{}` with connection parameters in the connection dialog: ```dotenv - DATABASE_HOST= - DATABASE_PORT=4000 - DATABASE_USER= - DATABASE_PASSWORD= - DATABASE_NAME=test - DATABASE_ENABLE_SSL=true + DATABASE_HOST={host} + DATABASE_PORT=4000 + DATABASE_USER={user} + DATABASE_PASSWORD={password} + DATABASE_NAME=test + DATABASE_ENABLE_SSL=true ``` > **Note** > - > For TiDB Serverless, TLS connection **MUST** be enabled via `DATABASE_ENABLE_SSL` when using public endpoint. + > For [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential), TLS connection **MUST** be enabled via `DATABASE_ENABLE_SSL` when using public endpoint. 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -132,28 +135,28 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele cp .env.example .env ``` -5. Edit the `.env` file, set up the environment variables as follows, and replace the corresponding placeholders `<>` with connection parameters in the connection dialog: +5. Edit the `.env` file, set up the environment variables as follows, and replace the corresponding placeholders `{}` with connection parameters in the connection dialog: ```dotenv - DATABASE_HOST= + DATABASE_HOST={host} DATABASE_PORT=4000 - DATABASE_USER= - DATABASE_PASSWORD= + DATABASE_USER={user} + DATABASE_PASSWORD={password} DATABASE_NAME=test DATABASE_ENABLE_SSL=true - DATABASE_SSL_CA= + DATABASE_SSL_CA={downloaded_ssl_ca_path} ``` > **Note** > - > It is recommended to enable TLS connection when using the public endpoint to connect to a TiDB Dedicated cluster. + > It is recommended to enable TLS connection when using the public endpoint to connect to a TiDB Cloud Dedicated cluster. > > To enable TLS connection, modify `DATABASE_ENABLE_SSL` to `true` and use `DATABASE_SSL_CA` to specify the file path of CA certificate downloaded from the connection dialog. 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -161,13 +164,13 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele cp .env.example .env ``` -2. Edit the `.env` file, set up the environment variables as follows, and replace the corresponding placeholders `<>` with your own TiDB connection information: +2. Edit the `.env` file, set up the environment variables as follows, and replace the corresponding placeholders `{}` with your own TiDB connection information: ```dotenv - DATABASE_HOST= + DATABASE_HOST={host} DATABASE_PORT=4000 - DATABASE_USER= - DATABASE_PASSWORD= + DATABASE_USER={user} + DATABASE_PASSWORD={password} DATABASE_NAME=test ``` @@ -189,7 +192,7 @@ ruby app.rb If the connection is successful, the console will output the version of the TiDB cluster as follows: ``` -🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v7.4.0) +🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v{{{ .tidb-version }}}) ⏳ Loading sample game data... ✅ Loaded sample game data. @@ -228,7 +231,7 @@ client = Mysql2::Client.new(options) > **Note** > -> For TiDB Serverless, TLS connection **MUST** be enabled via `DATABASE_ENABLE_SSL` when using public endpoint, but you **don't** have to specify an SSL CA certificate via `DATABASE_SSL_CA`, because mysql2 gem will search for existing CA certificates in a particular order until a file is discovered. +> For [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential), TLS connection **MUST** be enabled via `DATABASE_ENABLE_SSL` when using public endpoint, but you **don't** have to specify an SSL CA certificate via `DATABASE_SSL_CA`, because mysql2 gem will search for existing CA certificates in a particular order until a file is discovered. ### Insert data @@ -309,4 +312,14 @@ While it is possible to specify the CA certificate path manually, doing so might ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX) channel. \ No newline at end of file + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-sample-application-ruby-rails.md b/develop/dev-guide-sample-application-ruby-rails.md index 19fe96352b333..6863336c51d43 100644 --- a/develop/dev-guide-sample-application-ruby-rails.md +++ b/develop/dev-guide-sample-application-ruby-rails.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and Rails to accomplish the foll > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with {{{ .starter }}}, {{{ .essential }}}, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -30,13 +30,13 @@ To complete this tutorial, you need: -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a {{{ .starter }}} cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -78,15 +78,15 @@ bundle add mysql2 dotenv Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. In the connection dialog, select `Rails` from the **Connect With** drop-down list and keep the default setting of the **Endpoint Type** as `Public`. +3. In the connection dialog, select `Rails` from the **Connect With** drop-down list and keep the default setting of the **Connection Type** as `Public`. -4. If you have not set a password yet, click **Create password** to generate a random password. +4. If you have not set a password yet, click **Generate Password** to generate a random password. 5. Run the following command to copy `.env.example` and rename it to `.env`: @@ -97,25 +97,27 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Edit the `.env` file, set up the `DATABASE_URL` environment variable as follows, and copy the connection string from the connection dialog as the variable value. ```dotenv - DATABASE_URL=mysql2://:@:/?ssl_mode=verify_identity + DATABASE_URL='mysql2://{user}:{password}@{host}:{port}/{database_name}?ssl_mode=verify_identity' ``` > **Note** > - > For TiDB Serverless, TLS connection **MUST** be enabled with the `ssl_mode=verify_identity` query parameter when using public endpoint. + > For [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential), TLS connection **MUST** be enabled with the `ssl_mode=verify_identity` query parameter when using public endpoint. 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -126,19 +128,19 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 5. Edit the `.env` file, set up the `DATABASE_URL` environment variable as follows, copy the connection string from the connection dialog as the variable value, and set the `sslca` query parameter to the file path of the CA certificate downloaded from the connection dialog: ```dotenv - DATABASE_URL=mysql2://:@:/?ssl_mode=verify_identity&sslca=/path/to/ca.pem + DATABASE_URL='mysql2://{user}:{password}@{host}:{port}/{database}?ssl_mode=verify_identity&sslca=/path/to/ca.pem' ``` > **Note** > - > It is recommended to enable TLS connection when using the public endpoint to connect to TiDB Dedicated. + > It is recommended to enable TLS connection when using the public endpoint to connect to TiDB Cloud Dedicated. > > To enable TLS connection, modify the value of the `ssl_mode` query parameter to `verify_identity` and the value of `sslca` to the file path of CA certificate downloaded from the connection dialog. 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -146,10 +148,10 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele cp .env.example .env ``` -2. Edit the `.env` file, set up the `DATABASE_URL` environment variable as follows, and replace the ``, ``, ``, ``, and `` with your own TiDB connection information: +2. Edit the `.env` file, set up the `DATABASE_URL` environment variable as follows, and replace the `{user}`, `{password}`, `{host}`, `{port}`, and `{database}` with your own TiDB connection information: ```dotenv - DATABASE_URL=mysql2://:@:/ + DATABASE_URL='mysql2://{user}:{password}@{host}:{port}/{database}' ``` If you are running TiDB locally, the default host address is `127.0.0.1`, and the password is empty. @@ -183,7 +185,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele If the connection is successful, the console will output the version of the TiDB cluster as follows: ``` -🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v7.4.0) +🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v{{{ .tidb-version }}}) ⏳ Loading sample game data... ✅ Loaded sample game data. @@ -223,7 +225,7 @@ production: > **Note** > -> For TiDB Serverless, TLS connection **MUST** be enabled via setting the `ssl_mode` query parameter to `verify_identity` in `DATABASE_URL` when using public endpoint, but you **don't** have to specify an SSL CA certificate via `DATABASE_URL`, because mysql2 gem will search for existing CA certificates in a particular order until a file is discovered. +> For [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential), TLS connection **MUST** be enabled via setting the `ssl_mode` query parameter to `verify_identity` in `DATABASE_URL` when using public endpoint, but you **don't** have to specify an SSL CA certificate via `DATABASE_URL`, because mysql2 gem will search for existing CA certificates in a particular order until a file is discovered. ### Insert data @@ -284,4 +286,14 @@ While it is possible to specify the CA certificate path manually, this approach ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX) channel. \ No newline at end of file + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-schema-design-overview.md b/develop/dev-guide-schema-design-overview.md index cd2c47df16e56..53398aedb122e 100644 --- a/develop/dev-guide-schema-design-overview.md +++ b/develop/dev-guide-schema-design-overview.md @@ -88,3 +88,17 @@ As a best practice, it is recommended that you use a [MySQL client](https://dev. ## Object limitations For more information, see [TiDB Limitations](/tidb-limitations.md). + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-sql-development-specification.md b/develop/dev-guide-sql-development-specification.md index d0254495eb946..8f6c616b52189 100644 --- a/develop/dev-guide-sql-development-specification.md +++ b/develop/dev-guide-sql-development-specification.md @@ -52,3 +52,17 @@ This document introduces some general development specifications for using SQL. - [TiFlash supported push-down calculations](/tiflash/tiflash-supported-pushdown-calculations.md). - [TiKV - List of Expressions for Pushdown](/functions-and-operators/expressions-pushed-down.md). - [Predicate push down](/predicate-push-down.md). + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-third-party-support.md b/develop/dev-guide-third-party-support.md index 263b6212a788e..48be728ff5d8d 100644 --- a/develop/dev-guide-third-party-support.md +++ b/develop/dev-guide-third-party-support.md @@ -44,7 +44,7 @@ If you encounter problems when connecting to TiDB using the tools listed in this v1.6.0 Full N/A - Connect to TiDB with Go-MySQL-Driver + Connect to TiDB with Go-MySQL-Driver Java @@ -53,11 +53,11 @@ If you encounter problems when connecting to TiDB using the tools listed in this Full - Connect to TiDB with JDBC + Connect to TiDB with JDBC @@ -82,7 +82,7 @@ If you encounter problems when connecting to TiDB using the tools listed in this v1.23.5 Full N/A - Connect to TiDB with GORM + Connect to TiDB with GORM beego @@ -111,21 +111,21 @@ If you encounter problems when connecting to TiDB using the tools listed in this 6.1.0.Final Full N/A - Connect to TiDB with Hibernate + Connect to TiDB with Hibernate MyBatis v3.5.10 Full N/A - Connect to TiDB with MyBatis + Connect to TiDB with MyBatis Spring Data JPA 2.7.2 Full N/A - Connect to TiDB with Spring Boot + Connect to TiDB with Spring Boot jOOQ @@ -140,43 +140,68 @@ If you encounter problems when connecting to TiDB using the tools listed in this v7.0 Full N/A - N/A + Connect to TiDB with Rails Framework and ActiveRecord ORM - JavaScript / TypeScript - sequelize + JavaScript / TypeScript + Sequelize v6.20.1 Full N/A - N/A + Connect to TiDB with Sequelize - Prisma Client + Prisma 4.16.2 Full N/A + Connect to TiDB with Prisma + + + TypeORM + v0.3.17 + Full N/A + Connect to TiDB with TypeORM Python Django - v4.1 + v4.2 Full django-tidb - Connect to TiDB with Django + Connect to TiDB with Django SQLAlchemy v1.4.37 Full N/A - Connect to TiDB with SQLAlchemy + Connect to TiDB with SQLAlchemy ## GUI -| GUI | Latest tested version | Support level | Tutorial | -| - | - | - | - | -| [DBeaver](https://dbeaver.io/) | 23.0.3 | Full | [Connect to TiDB with DBeaver](/develop/dev-guide-gui-dbeaver.md) | +| GUI | Latest tested version | Support level | Tutorial | +|-----------------------------------------------------------|-----------------------|---------------|--------------------------------------------------------------------------------------| +| [Beekeeper Studio](https://www.beekeeperstudio.io/) | 4.3.0 | Full | N/A | +| [JetBrains DataGrip](https://www.jetbrains.com/datagrip/) | 2023.2.1 | Full | [Connect to TiDB with JetBrains DataGrip](/develop/dev-guide-gui-datagrip.md) | +| [DBeaver](https://dbeaver.io/) | 23.0.3 | Full | [Connect to TiDB with DBeaver](/develop/dev-guide-gui-dbeaver.md) | +| [Visual Studio Code](https://code.visualstudio.com/) | 1.72.0 | Full | [Connect to TiDB with Visual Studio Code](/develop/dev-guide-gui-vscode-sqltools.md) | +| [Navicat](https://www.navicat.com) | 17.1.6 | Full | [Connect to TiDB with Navicat](/develop/dev-guide-gui-navicat.md) | + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-third-party-tools-compatibility.md b/develop/dev-guide-third-party-tools-compatibility.md index 41f5d1078a5a3..1f7e611259eeb 100644 --- a/develop/dev-guide-third-party-tools-compatibility.md +++ b/develop/dev-guide-third-party-tools-compatibility.md @@ -153,27 +153,27 @@ To ensure data consistency by transaction, you can use `UPDATE` statements to up **Description** -When the `useLocalTransactionState` and `rewriteBatchedStatements` parameters are set to `true` at the same time, the transaction might fail to commit. You can reproduce with [this code](https://github.com/Icemap/tidb-java-gitpod/tree/reproduction-local-transaction-state-txn-error). +When using MySQL Connector/J 8.0.32 or an earlier version, if the `useLocalTransactionState` and `rewriteBatchedStatements` parameters are set to `true` at the same time, the transaction might fail to commit. You can reproduce with [this code](https://github.com/Icemap/tidb-java-gitpod/tree/reproduction-local-transaction-state-txn-error). **Way to avoid** > **Note:** > -> This bug has been reported to MySQL JDBC. To keep track of the process, you can follow this [Bug Report](https://bugs.mysql.com/bug.php?id=108643). +> `useConfigs=maxPerformance` includes a group of configurations. For detailed configurations in MySQL Connector/J 8.0 and MySQL Connector/J 5.1, see [mysql-connector-j 8.0](https://github.com/mysql/mysql-connector-j/blob/release/8.0/src/main/resources/com/mysql/cj/configurations/maxPerformance.properties) and [mysql-connector-j 5.1](https://github.com/mysql/mysql-connector-j/blob/release/5.1/src/com/mysql/jdbc/configs/maxPerformance.properties) respectively. You need to disable `useLocalTransactionState` when using `maxPerformance`. That is, use `useConfigs=maxPerformance&useLocalTransactionState=false`. -**DO NOT** turn on `useLocalTransactionState` as this might prevent transactions from being committed or rolled back. +This bug has been fixed in MySQL Connector/J 8.0.33. Considering updates for the 8.0.x series have ceased, it is strongly recommended to upgrade your MySQL Connector/J to [the latest General Availability (GA) version](https://dev.mysql.com/downloads/connector/j/) for improved stability and performance. ### Connector is incompatible with the server version earlier than 5.7.5 **Description** -The database connection might hang under certain conditions when using MySQL Connector/J 8.0.29 with a MySQL server < 5.7.5 or a database using the MySQL server < 5.7.5 protocol (such as TiDB earlier than v6.3.0). For more details, see the [Bug Report](https://bugs.mysql.com/bug.php?id=106252). +The database connection might hang under certain conditions when using MySQL Connector/J 8.0.31 or an earlier version with a MySQL server < 5.7.5 or a database using the MySQL server < 5.7.5 protocol (such as TiDB earlier than v6.3.0). For more details, see the [Bug Report](https://bugs.mysql.com/bug.php?id=106252). **Way to avoid** -This is a known issue. As of October 12, 2022, MySQL Connector/J has not fixed the issue. +This bug has been fixed in MySQL Connector/J 8.0.32. Considering updates for the 8.0.x series have ceased, it is strongly recommended to upgrade your MySQL Connector/J to [the latest General Availability (GA) version](https://dev.mysql.com/downloads/connector/j/) for improved stability and performance. -TiDB fixes it in the following ways: +TiDB also fixes it in the following ways: - Client side: This bug has been fixed in **pingcap/mysql-connector-j** and you can use the [pingcap/mysql-connector-j](https://github.com/pingcap/mysql-connector-j) instead of the official MySQL Connector/J. - Server side: This compatibility issue has been fixed since TiDB v6.3.0 and you can upgrade the server to v6.3.0 or later versions. @@ -231,3 +231,17 @@ To allow the removal of the `AUTO_INCREMENT` attribute, set `@@tidb_allow_remove **Description** `FULLTEXT`, `HASH`, and `SPATIAL` indexes are not supported. + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-tidb-crud-sql.md b/develop/dev-guide-tidb-crud-sql.md index 9fa4493e245f6..c866b94759068 100644 --- a/develop/dev-guide-tidb-crud-sql.md +++ b/develop/dev-guide-tidb-crud-sql.md @@ -1,15 +1,15 @@ --- title: CRUD SQL in TiDB -summary: A brief introduction to TiDB's CURD SQL. +summary: A brief introduction to TiDB's CRUD SQL. --- # CRUD SQL in TiDB -This document briefly introduces how to use TiDB's CURD SQL. +This document briefly introduces how to use TiDB's CRUD SQL. ## Before you start -Please make sure you are connected to a TiDB cluster. If not, refer to [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster) to create a TiDB Serverless cluster. +Please make sure you are connected to a TiDB cluster. If not, refer to [Build a {{{ .starter }}} Cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-cloud-cluster) to create a {{{ .starter }}} cluster. ## Explore SQL with TiDB @@ -101,3 +101,17 @@ Use the `WHERE` clause to filter all records that match the conditions and then ```sql SELECT * FROM person WHERE id < 5; ``` + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-timeouts-in-tidb.md b/develop/dev-guide-timeouts-in-tidb.md index 6cf44e68c518c..f2abba5928769 100644 --- a/develop/dev-guide-timeouts-in-tidb.md +++ b/develop/dev-guide-timeouts-in-tidb.md @@ -11,9 +11,54 @@ This document describes various timeouts in TiDB to help you troubleshoot errors TiDB's transaction implementation uses the MVCC (Multiple Version Concurrency Control) mechanism. When the newly written data overwrites the old data, the old data will not be replaced, but kept together with the newly written data. The versions are distinguished by the timestamp. TiDB uses the mechanism of periodic Garbage Collection (GC) to clean up the old data that is no longer needed. -By default, each MVCC version (consistency snapshots) is kept for 10 minutes. Transactions that take longer than 10 minutes to read will receive an error `GC life time is shorter than transaction duration`. +- For TiDB versions earlier than v4.0: -If you need longer read time, for example, when you are using **Dumpling** for full backups (**Dumpling** backs up consistent snapshots), you can adjust the value of `tikv_gc_life_time` in the `mysql.tidb` table in TiDB to increase the MVCC version retention time. Note that `tikv_gc_life_time` takes effect globally and immediately. Increasing the value will increase the life time of all existing snapshots, and decreasing it will immediately shorten the life time of all snapshots. Too many MVCC versions will impact TiKV's processing efficiency. So you need to change `tikv_gc_life_time` back to the previous setting in time after doing a full backup with **Dumpling**. + By default, each MVCC version (consistency snapshots) is kept for 10 minutes. Transactions that take longer than 10 minutes to read will receive an error `GC life time is shorter than transaction duration`. + +- For TiDB v4.0 and later versions: + + For running transactions that do not exceed a duration of 24 hours, garbage collection (GC) are blocked during the transaction execution. The error `GC life time is shorter than transaction duration` does not occur. + +If you need longer read time temporarily in some cases, you can increase the retention time of MVCC versions: + +- For TiDB versions earlier than v5.0: adjust `tikv_gc_life_time` in the `mysql.tidb` table in TiDB. +- For TiDB v5.0 and later versions: adjust the system variable [`tidb_gc_life_time`](/system-variables.md#tidb_gc_life_time-new-in-v50). + +Note that the system variable configuration takes effect globally and immediately. Increasing its value will increase the life time of all existing snapshots, and decreasing it will immediately shorten the life time of all snapshots. Too many MVCC versions will impact the performance of the TiDB cluster. So you need to change this variable back to the previous setting in time. + + + +> **Tip:** +> +> Specifically, when Dumpling is exporting data from TiDB (less than 1 TB), if the TiDB version is v4.0.0 or later and Dumpling can access the PD address and the [`INFORMATION_SCHEMA.CLUSTER_INFO`](/information-schema/information-schema-cluster-info.md) table of the TiDB cluster, Dumpling automatically adjusts the GC safe point to block GC without affecting the original cluster. +> +> However, in either of the following scenarios, Dumpling cannot automatically adjust the GC time: +> +> - The data size is very large (more than 1 TB). +> - Dumpling cannot connect directly to PD, for example, the TiDB cluster is on TiDB Cloud or on Kubernetes that is separated from Dumpling. +> +> In such scenarios, you must manually extend the GC time in advance to avoid export failure due to GC during the export process. +> +> For more details, see [Manually set the TiDB GC time](/dumpling-overview.md#manually-set-the-tidb-gc-time). + + + + + +> **Tip:** +> +> Specifically, when Dumpling is exporting data from TiDB (less than 1 TB), if the TiDB version is later than or equal to v4.0.0 and Dumpling can access the PD address of the TiDB cluster, Dumpling automatically extends the GC time without affecting the original cluster. +> +> However, in either of the following scenarios, Dumpling cannot automatically adjust the GC time: +> +> - The data size is very large (more than 1 TB). +> - Dumpling cannot connect directly to PD, for example, the TiDB cluster is on TiDB Cloud or on Kubernetes that is separated from Dumpling. +> +> In such scenarios, you must manually extend the GC time in advance to avoid export failure due to GC during the export process. +> +> For more details, see [Manually set the TiDB GC time](https://docs.pingcap.com/tidb/stable/dumpling-overview#manually-set-the-tidb-gc-time). + + For more information about GC, see [GC Overview](/garbage-collection-overview.md). @@ -29,7 +74,25 @@ TiDB also provides a system variable (`max_execution_time`, `0` by default, indi ## JDBC query timeout -MySQL JDBC's query timeout setting for `setQueryTimeout()` does **_NOT_** work for TiDB, because the client sends a `KILL` command to the database when it detects the timeout. However, the tidb-server is load balanced, and it will not execute this `KILL` command to avoid termination of the connection on a wrong tidb-server. You need to use `MAX_EXECUTION_TIME` to check the query timeout effect. + + +Starting from v6.1.0, when the [`enable-global-kill`](/tidb-configuration-file.md#enable-global-kill-new-in-v610) configuration item is set to its default value `true`, you can use the `setQueryTimeout()` method provided by MySQL JDBC to control the query timeout. + +> **Note:** +> +> When your TiDB version is earlier than v6.1.0 or [`enable-global-kill`](/tidb-configuration-file.md#enable-global-kill-new-in-v610) is set to `false`, `setQueryTimeout()` does not work for TiDB. This is because the client sends a `KILL` command to the database when it detects the query timeout. However, because the TiDB service is load balanced, TiDB does not execute the `KILL` command to avoid termination of the connection on a wrong TiDB node. In such cases, you can use `max_execution_time` to control query timeout. + + + + + +Starting from v6.1.0, when the [`enable-global-kill`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file/#enable-global-kill-new-in-v610) configuration item is set to its default value `true`, you can use the `setQueryTimeout()` method provided by MySQL JDBC to control the query timeout. + +> **Note:** +> +> When your TiDB version is earlier than v6.1.0 or [`enable-global-kill`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file/#enable-global-kill-new-in-v610) is set to `false`, `setQueryTimeout()` does not work for TiDB. This is because the client sends a `KILL` command to the database when it detects the query timeout. However, because the TiDB service is load balanced, TiDB does not execute the `KILL` command to avoid termination of the connection on a wrong TiDB node. In such cases, you can use `max_execution_time` to control query timeout. + + TiDB provides the following MySQL-compatible timeout control parameters. @@ -41,3 +104,17 @@ However, in a real production environment, idle connections and indefinitely exe - `sessionVariables=wait_timeout=3600` (1 hour) - `sessionVariables=max_execution_time=300000` (5 minutes) + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-transaction-overview.md b/develop/dev-guide-transaction-overview.md index 4e9358707726f..000971db7af16 100644 --- a/develop/dev-guide-transaction-overview.md +++ b/develop/dev-guide-transaction-overview.md @@ -158,4 +158,18 @@ mysql> SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; ERROR 8048 (HY000): The isolation level 'SERIALIZABLE' is not supported. Set tidb_skip_isolation_level_check=1 to skip this error ``` -TiDB implements Snapshot Isolation (SI) level consistency, also known as "repeatable read" for consistency with MySQL. This isolation level is different from [ANSI Repeatable Read Isolation Level](/transaction-isolation-levels.md#difference-between-tidb-and-ansi-repeatable-read) and [MySQL Repeatable Read Isolation Level](/transaction-isolation-levels.md#difference-between-tidb-and-mysql-repeatable-read). For more details, see [TiDB Transaction Isolation Levels](/transaction-isolation-levels.md). \ No newline at end of file +TiDB implements Snapshot Isolation (SI) level consistency, also known as "repeatable read" for consistency with MySQL. This isolation level is different from [ANSI Repeatable Read Isolation Level](/transaction-isolation-levels.md#difference-between-tidb-and-ansi-repeatable-read) and [MySQL Repeatable Read Isolation Level](/transaction-isolation-levels.md#difference-between-tidb-and-mysql-repeatable-read). For more details, see [TiDB Transaction Isolation Levels](/transaction-isolation-levels.md). + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-transaction-restraints.md b/develop/dev-guide-transaction-restraints.md index 6c180e6d29bf6..311d9e6530974 100644 --- a/develop/dev-guide-transaction-restraints.md +++ b/develop/dev-guide-transaction-restraints.md @@ -681,6 +681,10 @@ mysql> SELECT * FROM doctors; ## Support for `savepoint` and nested transactions +> **Note:** +> +> Starting from v6.2.0, TiDB supports the [`savepoint`](/sql-statements/sql-statement-savepoint.md) feature. If your TiDB cluster is earlier than v6.2.0, your TiDB cluster does not support the `PROPAGATION_NESTED` behavior. It is recommended to upgrade to v6.2.0 or a later version. If upgrading TiDB is not possible, and your applications are based on the **Java Spring** framework that uses the `PROPAGATION_NESTED` propagation behavior, you need to adapt it on the application side to remove the logic for nested transactions. + The `PROPAGATION_NESTED` propagation behavior supported by **Spring** triggers a nested transaction, which is a child transaction that is started independently of the current transaction. A `savepoint` is recorded when the nested transaction starts. If the nested transaction fails, the transaction will roll back to the `savepoint` state. The nested transaction is part of the outer transaction and will be committed together with the outer transaction. The following example demonstrates the `savepoint` mechanism: @@ -701,23 +705,37 @@ mysql> SELECT * FROM T2; +------+ ``` -> **Note:** -> -> Since v6.2.0, TiDB supports the `savepoint` feature. If your TiDB cluster is earlier than v6.2.0, your TiDB cluster does not support the `PROPAGATION_NESTED` behavior. If your applications are based on the **Java Spring** framework that use the `PROPAGATION_NESTED` propagation behavior, you need to adapt it on the application side to remove the logic for nested transactions. - ## Large transaction restrictions The basic principle is to limit the size of the transaction. At the KV level, TiDB has a restriction on the size of a single transaction. At the SQL level, one row of data is mapped to one KV entry, and each additional index will add one KV entry. The restriction is as follows at the SQL level: -- The maximum single row record size is `120 MB`. You can configure it by `performance.txn-entry-size-limit` for TiDB v5.0 and later versions. The value is `6 MB` for earlier versions. -- The maximum single transaction size supported is `10 GB`. You can configure it by `performance.txn-total-size-limit` for TiDB v4.0 and later versions. The value is `100 MB` for earlier versions. +- The maximum single row record size is 120 MiB. You can adjust it by using the [`performance.txn-entry-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-entry-size-limit-new-in-v4010-and-v500) configuration parameter of tidb-server for TiDB v4.0.10 and later v4.0.x versions, TiDB v5.0.0 and later versions. The value is `6 MB` for versions earlier than v4.0.10. + +- The maximum single transaction size supported is 1 TiB. + + - For TiDB v4.0 and later versions, You can configure it by [`performance.txn-total-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-total-size-limit)). The value is `100 MB` for earlier versions. + - For TiDB v6.5.0 and later versions, this configuration is no longer recommended. For more information, see [`performance.txn-total-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-total-size-limit)). Note that for both the size restrictions and row restrictions, you should also consider the overhead of encoding and additional keys for the transaction during the transaction execution. To achieve optimal performance, it is recommended to write one transaction every 100 ~ 500 rows. ## Auto-committed `SELECT FOR UPDATE` statements do NOT wait for locks -Currently locks are not added to auto-committed `SELECT FOR UPDATE` statements. The effect is shown in the following figure: +Currently, locks are not added to auto-committed `SELECT FOR UPDATE` statements. The following screenshot shows the effect in two separate sessions: ![The situation in TiDB](/media/develop/autocommit_selectforupdate_nowaitlock.png) This is a known incompatibility issue with MySQL. You can solve this issue by using the explicit `BEGIN;COMMIT;` statements. + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + diff --git a/develop/dev-guide-transaction-troubleshoot.md b/develop/dev-guide-transaction-troubleshoot.md index 03d37ceb22c67..e2c5a5bf6f84d 100644 --- a/develop/dev-guide-transaction-troubleshoot.md +++ b/develop/dev-guide-transaction-troubleshoot.md @@ -153,4 +153,18 @@ For information about how to troubleshoot and resolve transaction conflicts, see - [Troubleshoot Write Conflicts in Optimistic Transactions](https://docs.pingcap.com/tidb/stable/troubleshoot-write-conflicts) + + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + \ No newline at end of file diff --git a/develop/dev-guide-troubleshoot-overview.md b/develop/dev-guide-troubleshoot-overview.md index acc046c208d33..444e3aa9d492a 100644 --- a/develop/dev-guide-troubleshoot-overview.md +++ b/develop/dev-guide-troubleshoot-overview.md @@ -42,3 +42,17 @@ See [Handle transaction errors](/develop/dev-guide-transaction-troubleshoot.md). - [TiDB FAQs](/faq/tidb-faq.md) + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-unique-serial-number-generation.md b/develop/dev-guide-unique-serial-number-generation.md index 7fdb7ab8ef522..c8885610337ad 100644 --- a/develop/dev-guide-unique-serial-number-generation.md +++ b/develop/dev-guide-unique-serial-number-generation.md @@ -48,4 +48,18 @@ The number allocation solution can be understood as bulk acquisition of auto-inc Every time, the application gets a segment of sequence numbers at the configured step. It updates the database at the same time to persist the maximum value of the current sequence that has been allocated. The processing and allocation of sequence numbers are completed in the application's memory. After a segment of sequence numbers is used up, the application gets a new segment of sequence numbers, which effectively alleviates the pressure on the database write. In practice, you can also adjust the step to control the frequency of database updates. -Finally, note that the IDs generated by the above two solutions are not random enough to be directly used as **primary keys** for TiDB tables. In practice, you can perform bit-reverse on the generated IDs to get more random new IDs. +Finally, note that the IDs generated by the above two solutions are not random enough to be directly used as **primary keys** for TiDB tables. In practice, you can perform bit-reverse on the generated IDs to get more random new IDs. For example, after performing bit-reverse, the ID `00000010100101000001111010011100` becomes `00111001011110000010100101000000`, and `11111111111111111111111111111101` becomes `10111111111111111111111111111111`. + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-unstable-result-set.md b/develop/dev-guide-unstable-result-set.md index dacc60b3d332c..ec4db05fbe474 100644 --- a/develop/dev-guide-unstable-result-set.md +++ b/develop/dev-guide-unstable-result-set.md @@ -234,3 +234,17 @@ To let `GROUP_CONCAT()` get the result set output in order, you need to add the ## Unstable results in `SELECT * FROM T LIMIT N` The returned result is related to the distribution of data on the storage node (TiKV). If multiple queries are performed, different storage units (Regions) of the storage nodes (TiKV) return results at different speeds, which can cause unstable results. + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-update-data.md b/develop/dev-guide-update-data.md index 2dcb69a80ccbb..dcfc85e7dda9d 100644 --- a/develop/dev-guide-update-data.md +++ b/develop/dev-guide-update-data.md @@ -14,7 +14,7 @@ This document describes how to use the following SQL statements to update the da Before reading this document, you need to prepare the following: -- [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). +- [Build a {{{ .starter }}} Cluster](/develop/dev-guide-build-cluster-in-cloud.md). - Read [Schema Design Overview](/develop/dev-guide-schema-design-overview.md), [Create a Database](/develop/dev-guide-create-database.md), [Create a Table](/develop/dev-guide-create-table.md), and [Create Secondary Indexes](/develop/dev-guide-create-secondary-indexes.md). - If you want to `UPDATE` data, you need to [insert data](/develop/dev-guide-insert-data.md) first. @@ -447,3 +447,17 @@ In each iteration, `SELECT` queries in order of the primary key. It selects prim
+ +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-use-common-table-expression.md b/develop/dev-guide-use-common-table-expression.md index f9b89a6da9f31..c8cd943efce80 100644 --- a/develop/dev-guide-use-common-table-expression.md +++ b/develop/dev-guide-use-common-table-expression.md @@ -214,3 +214,17 @@ The result is as follows: ## Read more - [WITH](/sql-statements/sql-statement-with.md) + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-use-follower-read.md b/develop/dev-guide-use-follower-read.md index 4571756bb3337..b2ad9b21bf6a8 100644 --- a/develop/dev-guide-use-follower-read.md +++ b/develop/dev-guide-use-follower-read.md @@ -160,3 +160,17 @@ public static class AuthorDAO { - [TiDB Cloud Key Visualizer Page](/tidb-cloud/tune-performance.md#key-visualizer) + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-use-stale-read.md b/develop/dev-guide-use-stale-read.md index fbdb8654cd33f..8817ee44ae3ae 100644 --- a/develop/dev-guide-use-stale-read.md +++ b/develop/dev-guide-use-stale-read.md @@ -497,3 +497,17 @@ public static class StaleReadHelper{ - [Usage Scenarios of Stale Read](/stale-read.md) - [Read Historical Data Using the `AS OF TIMESTAMP` Clause](/as-of-timestamp.md) - [Read Historical Data Using the `tidb_read_staleness` System Variable](/tidb-read-staleness.md) + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-use-subqueries.md b/develop/dev-guide-use-subqueries.md index 1eb078c5ea9a2..ae8d3a6346e53 100644 --- a/develop/dev-guide-use-subqueries.md +++ b/develop/dev-guide-use-subqueries.md @@ -127,4 +127,18 @@ As a best practice, in actual development, it is recommended to avoid querying t - [Subquery Related Optimizations](/subquery-optimization.md) - [Decorrelation of Correlated Subquery](/correlated-subquery-optimization.md) -- [Subquery Optimization in TiDB](https://en.pingcap.com/blog/subquery-optimization-in-tidb/) +- [Subquery Optimization in TiDB](https://www.pingcap.com/blog/subquery-optimization-in-tidb/) + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-use-temporary-tables.md b/develop/dev-guide-use-temporary-tables.md index 279a3dbab8e27..cbd9e5cc401c3 100644 --- a/develop/dev-guide-use-temporary-tables.md +++ b/develop/dev-guide-use-temporary-tables.md @@ -257,3 +257,17 @@ For limitations of temporary tables in TiDB, see [Compatibility restrictions wit ## Read more - [Temporary Tables](/temporary-tables.md) + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-use-views.md b/develop/dev-guide-use-views.md index 01143079f5f00..3480814af215d 100644 --- a/develop/dev-guide-use-views.md +++ b/develop/dev-guide-use-views.md @@ -122,3 +122,17 @@ For limitations of views in TiDB, see [Limitations of Views](/views.md#limitatio - [DROP VIEW Statement](/sql-statements/sql-statement-drop-view.md) - [EXPLAIN Statements Using Views](/explain-views.md) - [TiFlink: Strongly Consistent Materialized Views Using TiKV and Flink](https://github.com/tiflink/tiflink) + +## Need help? + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). + + + + + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). + + \ No newline at end of file diff --git a/best-practices/java-app-best-practices.md b/develop/java-app-best-practices.md similarity index 82% rename from best-practices/java-app-best-practices.md rename to develop/java-app-best-practices.md index 776aa57bba6c3..ca164d34ce5b5 100644 --- a/best-practices/java-app-best-practices.md +++ b/develop/java-app-best-practices.md @@ -1,7 +1,6 @@ --- title: Best Practices for Developing Java Applications with TiDB -summary: Learn the best practices for developing Java applications with TiDB. -aliases: ['/docs/dev/best-practices/java-app-best-practices/','/docs/dev/reference/best-practices/java-app/'] +summary: This document introduces best practices for developing Java applications with TiDB, covering database-related components, JDBC usage, connection pool configuration, data access framework, Spring Transaction, and troubleshooting tools. TiDB is highly compatible with MySQL, so most MySQL-based Java application best practices also apply to TiDB. --- # Best Practices for Developing Java Applications with TiDB @@ -13,7 +12,7 @@ This document introduces the best practice for developing Java applications to b Common components that interact with the TiDB database in Java applications include: - Network protocol: A client interacts with a TiDB server via the standard [MySQL protocol](https://dev.mysql.com/doc/dev/mysql-server/latest/PAGE_PROTOCOL.html). -- JDBC API and JDBC drivers: Java applications usually use the standard [JDBC (Java Database Connectivity)](https://docs.oracle.com/javase/8/docs/technotes/guides/jdbc/) API to access a database. To connect to TiDB, you can use a JDBC driver that implements the MySQL protocol via the JDBC API. Such common JDBC drivers for MySQL include [MySQL Connector/J](https://github.com/mysql/mysql-connector-j) and [MariaDB Connector/J](https://mariadb.com/kb/en/library/about-mariadb-connector-j/#about-mariadb-connectorj). +- JDBC API and JDBC drivers: Java applications usually use the standard [JDBC (Java Database Connectivity)](https://docs.oracle.com/javase/8/docs/technotes/guides/jdbc/) API to access a database. To connect to TiDB, you can use a JDBC driver that implements the MySQL protocol via the JDBC API. Such common JDBC drivers for MySQL include [MySQL Connector/J](https://github.com/mysql/mysql-connector-j) and [MariaDB Connector/J](https://mariadb.com/docs/connectors/mariadb-connector-j/about-mariadb-connector-j#about-mariadb-connectorj). - Database connection pool: To reduce the overhead of creating a connection each time it is requested, applications usually use a connection pool to cache and reuse connections. JDBC [DataSource](https://docs.oracle.com/javase/8/docs/api/javax/sql/DataSource.html) defines a connection pool API. You can choose from different open-source connection pool implementations as needed. - Data access framework: Applications usually use a data access framework such as [MyBatis](https://mybatis.org/mybatis-3/index.html) and [Hibernate](https://hibernate.org/) to further simplify and manage the database access operations. - Application implementation: The application logic controls when to send what commands to the database. Some applications use [Spring Transaction](https://docs.spring.io/spring/docs/4.2.x/spring-framework-reference/html/transaction.html) aspects to manage transactions' start and commit logics. @@ -76,7 +75,7 @@ TiDB supports both methods, but it is preferred that you use the first method, b ### MySQL JDBC parameters -JDBC usually provides implementation-related configurations in the form of JDBC URL parameters. This section introduces [MySQL Connector/J's parameter configurations](https://dev.mysql.com/doc/connector-j/en/connector-j-reference-configuration-properties.html) (If you use MariaDB, see [MariaDB's parameter configurations](https://mariadb.com/kb/en/library/about-mariadb-connector-j/#optional-url-parameters)). Because this document cannot cover all configuration items, it mainly focuses on several parameters that might affect performance. +JDBC usually provides implementation-related configurations in the form of JDBC URL parameters. This section introduces [MySQL Connector/J's parameter configurations](https://dev.mysql.com/doc/connector-j/en/connector-j-reference-configuration-properties.html) (If you use MariaDB, see [MariaDB's parameter configurations](https://mariadb.com/docs/connectors/mariadb-connector-j/about-mariadb-connector-j#optional-url-parameters)). Because this document cannot cover all configuration items, it mainly focuses on several parameters that might affect performance. #### Prepare-related parameters @@ -200,30 +199,71 @@ Through monitoring, you might notice that although the application only performs After it is configured, you can check the monitoring to see a decreased number of `SELECT` statements. +> **Note:** +> +> Enabling `useConfigs=maxPerformance` requires MySQL Connector/J version 8.0.33 or later. For more details, see [MySQL JDBC Bug](/develop/dev-guide-third-party-tools-compatibility.md#mysql-jdbc-bugs). + #### Timeout-related parameters TiDB provides two MySQL-compatible parameters that controls the timeout: `wait_timeout` and `max_execution_time`. These two parameters respectively control the connection idle timeout with the Java application and the timeout of the SQL execution in the connection; that is to say, these parameters control the longest idle time and the longest busy time for the connection between TiDB and the Java application. The default value of both parameters is `0`, which by default allows the connection to be infinitely idle and infinitely busy (an infinite duration for one SQL statement to execute). However, in an actual production environment, idle connections and SQL statements with excessively long execution time negatively affect databases and applications. To avoid idle connections and SQL statements that are executed for too long, you can configure these two parameters in your application's connection string. For example, set `sessionVariables=wait_timeout=3600` (1 hour) and `sessionVariables=max_execution_time=300000` (5 minutes). +#### Typical JDBC connection string parameters + +Combining the preceding parameter values, the JDBC connection string configuration is as follows: + +``` +jdbc:mysql://:/?characterEncoding=UTF-8&useSSL=false&useServerPrepStmts=true&cachePrepStmts=true&prepStmtCacheSqlLimit=10000&prepStmtCacheSize=1000&useConfigs=maxPerformance&rewriteBatchedStatements=true +``` + +> **Note:** +> +> If you are connecting over a public network, you need to set `useSSL=true` and [enable TLS between TiDB clients and servers](/enable-tls-between-clients-and-servers.md). + ## Connection pool Building TiDB (MySQL) connections is relatively expensive (for OLTP scenarios at least), because in addition to building a TCP connection, connection authentication is also required. Therefore, the client usually saves the TiDB (MySQL) connections to the connection pool for reuse. -Java has many connection pool implementations such as [HikariCP](https://github.com/brettwooldridge/HikariCP), [tomcat-jdbc](https://tomcat.apache.org/tomcat-10.1-doc/jdbc-pool.html), [druid](https://github.com/alibaba/druid), [c3p0](https://www.mchange.com/projects/c3p0/), and [dbcp](https://commons.apache.org/proper/commons-dbcp/). TiDB does not limit which connection pool you use, so you can choose whichever you like for your application. +TiDB supports the following Java connection pools: + +- [HikariCP](https://github.com/brettwooldridge/HikariCP) +- [tomcat-jdbc](https://tomcat.apache.org/tomcat-10.1-doc/jdbc-pool) +- [druid](https://github.com/alibaba/druid) +- [c3p0](https://www.mchange.com/projects/c3p0/) +- [dbcp](https://commons.apache.org/proper/commons-dbcp/) -### Configure the number of connections +In practice, some connection pools might persistently use specific active sessions. Although the total number of connections appears evenly distributed across TiDB compute nodes, uneven distribution of active connections can lead to actual load imbalance. In distributed scenarios, it is recommended to use HikariCP, which manages connection lifecycles effectively and helps prevent active connections from being fixed on certain nodes, achieving balanced load distribution. -It is a common practice that the connection pool size is well adjusted according to the application's own needs. Take HikariCP as an example: +### Typical connection pool configuration -- `maximumPoolSize`: The maximum number of connections in the connection pool. If this value is too large, TiDB consumes resources to maintain useless connections. If this value is too small, the application gets slow connections. So configure this value for your own good. For details, see [About Pool Sizing](https://github.com/brettwooldridge/HikariCP/wiki/About-Pool-Sizing). -- `minimumIdle`: The minimum number of idle connections in the connection pool. It is mainly used to reserve some connections to respond to sudden requests when the application is idle. You can also configure it according to your application needs. +The following is an example configuration for HikariCP: + +```yaml +hikari: + maximumPoolSize: 20 + poolName: hikariCP + connectionTimeout: 30000 + maxLifetime: 1200000 + keepaliveTime: 120000 +``` -The application needs to return the connection after finishing using it. It is also recommended that the application use the corresponding connection pool monitoring (such as `metricRegistry`) to locate the connection pool issue in time. +The parameter explanations are as follows. For more details, refer to the [official HikariCP documentation](https://github.com/brettwooldridge/HikariCP/blob/dev/README.md). + +- `maximumPoolSize`: the maximum number of connections in the pool. The default value is `10`. In containerized environments, it is recommended to set this to 4–10 times the number of CPU cores available to the Java application. Setting this value too high can lead to resource wastage, while setting it too low can slow down connection acquisition. See [About Pool Sizing](https://github.com/brettwooldridge/HikariCP/wiki/About-Pool-Sizing) for more details. +- `minimumIdle`: HikariCP recommends not setting this parameter. The default value is equal to the value of `maximumPoolSize`, which disables connection pool scaling. This ensures that connections are readily available during traffic spikes and avoids delays caused by connection creation. +- `connectionTimeout`: the maximum time (in milliseconds) that an application waits to acquire a connection from the pool. The default value is `30000` milliseconds (30 seconds). If no available connection is obtained within this time, a `SQLException` exception occurs. +- `maxLifetime`: the maximum lifetime (in milliseconds) of a connection in the pool. The default value is `1800000` milliseconds (30 minutes). Connections in use are not affected. After the connection is closed, it will be removed according to this setting. Setting this value too low can cause frequent reconnections. If you are using [`graceful-wait-before-shutdown`](/tidb-configuration-file.md#graceful-wait-before-shutdown-new-in-v50), ensure this value is less than the wait time. +- `keepaliveTime`: the interval (in milliseconds) between keepalive operations on connections in the pool. This setting helps prevent disconnections caused by database or network idle timeouts. The default value is `120000` milliseconds (2 minutes). The pool prefers using the JDBC4 `isValid()` method to keep idle connections alive. ### Probe configuration -The connection pool maintains persistent connections to TiDB. TiDB does not proactively close client connections by default (unless an error is reported), but generally there will be network proxies such as LVS or HAProxy between the client and TiDB. Usually, these proxies will proactively clean up connections that are idle for a certain period of time (controlled by the proxy's idle configuration). In addition to paying attention to the idle configuration of the proxies, the connection pool also needs to keep alive or probe connections. +The connection pool maintains persistent connections from clients to TiDB as follows: + +- Before v5.4, TiDB does not proactively close client connections by default (unless an error is reported). +- Starting from v5.4, TiDB automatically closes client connections after `28800` seconds (this is, `8` hours) of inactivity by default. You can control this timeout setting using the TiDB and MySQL compatible `wait_timeout` variable. For more information, see [JDBC Query Timeout](/develop/dev-guide-timeouts-in-tidb.md#jdbc-query-timeout). + +Moreover, there might be network proxies such as [LVS](https://en.wikipedia.org/wiki/Linux_Virtual_Server) or [HAProxy](https://en.wikipedia.org/wiki/HAProxy) between clients and TiDB. These proxies typically proactively clean up connections after a specific idle period (determined by the proxy's idle configuration). In addition to monitoring the proxy's idle configuration, connection pools also need to maintain or probe connections for keep-alive. If you often see the following error in your Java application: @@ -365,4 +405,6 @@ Obtaining flame graphs in Java applications is tedious. For details, see [Java F Based on commonly used Java components that interact with databases, this document describes the common problems and solutions for developing Java applications with TiDB. TiDB is highly compatible with the MySQL protocol, so most of the best practices for MySQL-based Java applications also apply to TiDB. -Join us at [TiDB Community slack channel](https://tidbcommunity.slack.com/archives/CH7TTLL7P), and share with broad TiDB user group about your experience or problems when you develop Java applications with TiDB. +## Need help? + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](/support.md). \ No newline at end of file diff --git a/dm/deploy-a-dm-cluster-using-binary.md b/dm/deploy-a-dm-cluster-using-binary.md index 17e9b26f55fce..8e94e724aeee3 100644 --- a/dm/deploy-a-dm-cluster-using-binary.md +++ b/dm/deploy-a-dm-cluster-using-binary.md @@ -1,7 +1,6 @@ --- title: Deploy Data Migration Using DM Binary summary: Learn how to deploy a Data Migration cluster using DM binary. -aliases: ['/docs/tidb-data-migration/dev/deploy-a-dm-cluster-using-binary/'] --- # Deploy Data Migration Using DM Binary diff --git a/dm/deploy-a-dm-cluster-using-tiup-offline.md b/dm/deploy-a-dm-cluster-using-tiup-offline.md index 69c70e862bb88..16a4f464f1784 100644 --- a/dm/deploy-a-dm-cluster-using-tiup-offline.md +++ b/dm/deploy-a-dm-cluster-using-tiup-offline.md @@ -46,7 +46,7 @@ This document describes how to deploy a DM cluster offline using TiUP. tiup mirror clone tidb-dm-${version}-linux-amd64 --os=linux --arch=amd64 \ --dm-master=${version} --dm-worker=${version} --dmctl=${version} \ --alertmanager=v0.17.0 --grafana=v4.0.3 --prometheus=v4.0.3 \ - --tiup=v$(tiup --version|grep 'tiup'|awk -F ' ' '{print $1}') --dm=v$(tiup --version|grep 'tiup'|awk -F ' ' '{print $1}') + --dm=v$(tiup --version|grep 'tiup'|awk -F ' ' '{print $1}') ``` The command above creates a directory named `tidb-dm-${version}-linux-amd64` in the current directory, which contains the component package managed by TiUP. @@ -127,7 +127,7 @@ alertmanager_servers: > > - Use `.` to indicate the subcategory of the configuration, such as `log.slow-threshold`. For more formats, see [TiUP configuration template](https://github.com/pingcap/tiup/blob/master/embed/examples/dm/topology.example.yaml). > -> - For more parameter description, see [master `config.toml.example`](https://github.com/pingcap/dm/blob/master/dm/master/dm-master.toml) and [worker `config.toml.example`](https://github.com/pingcap/dm/blob/master/dm/worker/dm-worker.toml). +> - For more parameter description, see [master `config.toml.example`](https://github.com/pingcap/tiflow/blob/master/dm/master/dm-master.toml) and [worker `config.toml.example`](https://github.com/pingcap/tiflow/blob/master/dm/worker/dm-worker.toml). > > - Make sure that the ports among the following components are interconnected: > - The `peer_port` (`8291` by default) among the DM-master nodes are interconnected. diff --git a/dm/deploy-a-dm-cluster-using-tiup.md b/dm/deploy-a-dm-cluster-using-tiup.md index 2d9e061141c6b..17413e153ddc7 100644 --- a/dm/deploy-a-dm-cluster-using-tiup.md +++ b/dm/deploy-a-dm-cluster-using-tiup.md @@ -1,7 +1,6 @@ --- title: Deploy a DM Cluster Using TiUP summary: Learn how to deploy TiDB Data Migration using TiUP DM. -aliases: ['/docs/tidb-data-migration/dev/deploy-a-dm-cluster-using-ansible/','/docs/tools/dm/deployment/'] --- # Deploy a DM Cluster Using TiUP @@ -144,7 +143,7 @@ alertmanager_servers: > - The TiUP nodes can connect to the `port` of all DM-master nodes (`8261` by default). > - The TiUP nodes can connect to the `port` of all DM-worker nodes (`8262` by default). -For more `master_servers.host.config` parameter description, refer to [master parameter](https://github.com/pingcap/dm/blob/master/dm/master/dm-master.toml). For more `worker_servers.host.config` parameter description, refer to [worker parameter](https://github.com/pingcap/dm/blob/master/dm/worker/dm-worker.toml). +For more `master_servers.host.config` parameter description, refer to [master parameter](https://github.com/pingcap/tiflow/blob/master/dm/master/dm-master.toml). For more `worker_servers.host.config` parameter description, refer to [worker parameter](https://github.com/pingcap/tiflow/blob/master/dm/worker/dm-worker.toml). ## Step 3: Execute the deployment command diff --git a/dm/dm-arch.md b/dm/dm-arch.md index 46ce7826a4d84..77804af759160 100644 --- a/dm/dm-arch.md +++ b/dm/dm-arch.md @@ -1,5 +1,6 @@ --- title: Data Migration Architecture +summary: "Data Migration (DM) architecture consists of three components: DM-master, DM-worker, and dmctl. DM-master manages data migration tasks, DM-worker executes specific tasks, and dmctl is a command line tool for cluster control. High availability is achieved through multiple DM-master nodes and automatic task scheduling. Full export and import tasks do not support high availability due to limitations in MySQL and DM-worker." --- # Data Migration Architecture diff --git a/dm/dm-best-practices.md b/dm/dm-best-practices.md index dddfb81835afe..16d8ccca4a90f 100644 --- a/dm/dm-best-practices.md +++ b/dm/dm-best-practices.md @@ -5,7 +5,7 @@ summary: Learn about best practices when you use TiDB Data Migration (DM) to mig # TiDB Data Migration (DM) Best Practices -[TiDB Data Migration (DM)](https://github.com/pingcap/tiflow/tree/master/dm) is a data migration tool developed by PingCAP. It supports full and incremental data migration from MySQL-compatible databases such as MySQL, Percona MySQL, MariaDB, Amazon RDS for MySQL, and Amazon Aurora into TiDB. +[TiDB Data Migration (DM)](https://github.com/pingcap/tiflow/tree/release-7.5/dm) is a data migration tool developed by PingCAP. It supports full and incremental data migration from MySQL-compatible databases such as MySQL, Percona MySQL, MariaDB, Amazon RDS for MySQL, and Amazon Aurora into TiDB. You can use DM in the following scenarios: @@ -61,11 +61,11 @@ When you create a table, you can declare that the primary key is either a cluste - Clustered indexes + `AUTO_RANDOM` - This solution can retain the benefits of using clustered indexes and avoid the write hotspot problem. It requires less effort for customization. You can modify the schema attribute when you switch to use TiDB as the write database. In subsequent queries, if you have to use the ID column to sort data, you can use the [`AUTO_RANDOM`](/auto-random.md) ID column and left shift 5 bits to ensure the order of the query data. For example: + This solution can retain the benefits of using clustered indexes and avoid the write hotspot problem. It requires less effort for customization. You can modify the schema attribute when you switch to use TiDB as the write database. In subsequent queries, if you have to use the ID column to sort data, you can use the [`AUTO_RANDOM`](/auto-random.md) ID column and left shift 6 bits (1 sign bit + 5 shard bits) to ensure the order of the query data. For example: ```sql CREATE TABLE t (a bigint PRIMARY KEY AUTO_RANDOM, b varchar(255)); - Select a, a<<5 ,b from t order by a <<5 desc + Select a, a<<6 ,b from t order by a <<6 desc ``` The following table summarizes the pros and cons of each solution. @@ -99,7 +99,7 @@ The following table summarizes the pros and cons of optimistic mode and pessimis | Scenario | Pros | Cons | | :--- | :--- | :--- | -| Pessimistic mode (Default) | It can ensure that the data migrated to the downstream will not go wrong. | If there are a large number of shards, the migration task will be blocked for a long time, or even stop if the upstream binlogs have been cleaned up. You can enable the relay log to avoid this problem. For more information, see [Use the relay log](#use-the relay-log). | +| Pessimistic mode (Default) | It can ensure that the data migrated to the downstream will not go wrong. | If there are a large number of shards, the migration task will be blocked for a long time, or even stop if the upstream binlogs have been cleaned up. You can enable the relay log to avoid this problem. For more information, see [Use the relay log](#use-the-relay-log). | | Optimistic mode| Upstream schema changes will not cause data migration latency. | In this mode, ensure that schema changes are compatible (check whether the incremental column has a default value). It is possible that the inconsistent data can be overlooked. For more information, see [Merge and Migrate Data from Sharded Tables in Optimistic Mode](/dm/feature-shard-merge-optimistic.md#restrictions).| ### Other restrictions and impact diff --git a/dm/dm-compatibility-catalog.md b/dm/dm-compatibility-catalog.md index a4a83a6ba2e68..ae634b4d1b6a3 100644 --- a/dm/dm-compatibility-catalog.md +++ b/dm/dm-compatibility-catalog.md @@ -32,7 +32,9 @@ DM supports migrating data from different sources to TiDB clusters. Based on the |Target database|Compatibility level|DM version| |-|-|-| -|TiDB 6.0|GA|≥ 5.3.1| +|TiDB 8.x|GA|≥ 5.3.1| +|TiDB 7.x|GA|≥ 5.3.1| +|TiDB 6.x|GA|≥ 5.3.1| |TiDB 5.4|GA|≥ 5.3.1| |TiDB 5.3|GA|≥ 5.3.1| |TiDB 5.2|GA|≥ 2.0.7, recommended: 5.4| @@ -41,4 +43,4 @@ DM supports migrating data from different sources to TiDB clusters. Based on the |TiDB 4.x|GA|≥ 2.0.1, recommended: 2.0.7| |TiDB 3.x|GA|≥ 2.0.1, recommended: 2.0.7| |MySQL|Experimental|| -|MariaDB|Experimental|| \ No newline at end of file +|MariaDB|Experimental|| diff --git a/dm/dm-config-overview.md b/dm/dm-config-overview.md index 449a836fee8c8..118067df24373 100644 --- a/dm/dm-config-overview.md +++ b/dm/dm-config-overview.md @@ -1,7 +1,6 @@ --- title: Data Migration Configuration File Overview summary: This document gives an overview of Data Migration configuration files. -aliases: ['/docs/tidb-data-migration/dev/config-overview/'] --- # Data Migration Configuration File Overview diff --git a/dm/dm-daily-check.md b/dm/dm-daily-check.md index c410c5dd99f7b..0376d91544d62 100644 --- a/dm/dm-daily-check.md +++ b/dm/dm-daily-check.md @@ -1,7 +1,6 @@ --- title: Daily Check for TiDB Data Migration summary: Learn about the daily check of TiDB Data Migration (DM). -aliases: ['/docs/tidb-data-migration/dev/daily-check/'] --- # Daily Check for TiDB Data Migration diff --git a/dm/dm-enable-tls.md b/dm/dm-enable-tls.md index 37dc390701cc9..292be260a1580 100644 --- a/dm/dm-enable-tls.md +++ b/dm/dm-enable-tls.md @@ -109,7 +109,7 @@ This section introduces how to enable encrypted data transmission between DM com ### Enable encrypted data transmission for downstream TiDB -1. Configure the downstream TiDB to use encrypted connections. For detailed operatons, refer to [Configure TiDB server to use secure connections](/enable-tls-between-clients-and-servers.md#configure-tidb-server-to-use-secure-connections). +1. Configure the downstream TiDB to use encrypted connections. For detailed operations, refer to [Configure TiDB server to use secure connections](/enable-tls-between-clients-and-servers.md#configure-tidb-server-to-use-secure-connections). 2. Set the TiDB client certificate in the task configuration file: diff --git a/dm/dm-error-handling.md b/dm/dm-error-handling.md index 6069d8a255233..879498880e8ad 100644 --- a/dm/dm-error-handling.md +++ b/dm/dm-error-handling.md @@ -1,7 +1,6 @@ --- title: Handle Errors in TiDB Data Migration summary: Learn about the error system and how to handle common errors when you use DM. -aliases: ['/docs/tidb-data-migration/dev/error-handling/','/docs/tidb-data-migration/dev/troubleshoot-dm/','/docs/tidb-data-migration/dev/error-system/'] --- # Handle Errors in TiDB Data Migration @@ -70,7 +69,7 @@ In the error system, usually, the information of a specific error is as follows: Whether DM outputs the error stack information depends on the error severity and the necessity. The error stack records the complete stack call information when the error occurs. If you cannot figure out the error cause based on the basic information and the error message, you can trace the execution path of the code when the error occurs using the error stack. -For the complete list of error codes, refer to the [error code lists](https://github.com/pingcap/dm/blob/master/_utils/terror_gen/errors_release.txt). +For the complete list of error codes, refer to the [error code lists](https://github.com/pingcap/tiflow/blob/master/dm/_utils/terror_gen/errors_release.txt). ## Troubleshooting diff --git a/dm/dm-faq.md b/dm/dm-faq.md index 12e5821c848c0..ae3c9547c2a9b 100644 --- a/dm/dm-faq.md +++ b/dm/dm-faq.md @@ -1,7 +1,6 @@ --- title: TiDB Data Migration FAQs summary: Learn about frequently asked questions (FAQs) about TiDB Data Migration (DM). -aliases: ['/docs/tidb-data-migration/dev/faq/'] --- # TiDB Data Migration FAQs @@ -25,7 +24,7 @@ Currently, DM does not support it and only supports the regular expressions of t ## If a statement executed upstream contains multiple DDL operations, does DM support such migration? -DM will attempt to split a single statement containing multiple DDL change operations into multiple statements containing only one DDL operation, but might not cover all cases. It is recommended to include only one DDL operation in a statement executed upstream, or verify it in the test environment. If it is not supported, you can file an [issue](https://github.com/pingcap/dm/issues) to the DM repository. +DM will attempt to split a single statement containing multiple DDL change operations into multiple statements containing only one DDL operation, but might not cover all cases. It is recommended to include only one DDL operation in a statement executed upstream, or verify it in the test environment. If it is not supported, you can file an [issue](https://github.com/pingcap/tiflow/issues) to the `pingcap/tiflow` repository. ## How to handle incompatible DDL statements? @@ -55,7 +54,7 @@ When an exception occurs during data migration and the data migration task canno ## How to handle the error returned by the DDL operation related to the gh-ost table, after `online-ddl: true` is set? ``` -[unit=Sync] ["error information"="{\"msg\":\"[code=36046:class=sync-unit:scope=internal:level=high] online ddls on ghost table `xxx`.`_xxxx_gho`\\ngithub.com/pingcap/dm/pkg/terror.(*Error).Generate ...... +[unit=Sync] ["error information"="{\"msg\":\"[code=36046:class=sync-unit:scope=internal:level=high] online ddls on ghost table `xxx`.`_xxxx_gho`\\ngithub.com/pingcap/tiflow/pkg/terror.(*Error).Generate ...... ``` The above error can be caused by the following reason: @@ -243,7 +242,7 @@ This is a known bug of TiUP, which is fixed in TiUP v1.3.2. The following are tw 2. Scale in and then scale out Grafana nodes in the cluster to restart the Grafana service. - Solution two: 1. Back up the `deploy/grafana-$port/bin/public` folder. - 2. Download the [TiUP DM offline package](https://download.pingcap.org/tidb-dm-v2.0.1-linux-amd64.tar.gz) and unpack it. + 2. Download the [TiUP DM offline package](https://download.pingcap.com/tidb-dm-v2.0.1-linux-amd64.tar.gz) and unpack it. 3. Unpack the `grafana-v4.0.3-**.tar.gz` in the offline package. 4. Replace the folder `deploy/grafana-$port/bin/public` with the `public` folder in `grafana-v4.0.3-**.tar.gz`. 5. Execute `tiup dm restart $cluster_name -R grafana` to restart the Grafana service. @@ -365,7 +364,7 @@ To solve this issue, you are recommended to maintain DM clusters using TiUP. In ## Why DM-master cannot be connected when I use dmctl to execute commands? -When using dmctl execute commands, you might find the connection to DM master fails (even if you have specified the parameter value of `--master-addr` in the command), and the error message is like `RawCause: context deadline exceeded, Workaround: please check your network connection.`. But afer checking the network connection using commands like `telnet `, no exception is found. +When using dmctl execute commands, you might find the connection to DM master fails (even if you have specified the parameter value of `--master-addr` in the command), and the error message is like `RawCause: context deadline exceeded, Workaround: please check your network connection.`. But after checking the network connection using commands like `telnet `, no exception is found. In this case, you can check the environment variable `https_proxy` (note that it is **https**). If this variable is configured, dmctl automatically connects the host and port specified by `https_proxy`. If the host does not have a corresponding `proxy` forwarding service, the connection fails. diff --git a/dm/dm-glossary.md b/dm/dm-glossary.md index a92a834e43a4d..3fca6c0475247 100644 --- a/dm/dm-glossary.md +++ b/dm/dm-glossary.md @@ -1,7 +1,6 @@ --- title: TiDB Data Migration Glossary summary: Learn the terms used in TiDB Data Migration. -aliases: ['/docs/tidb-data-migration/dev/glossary/'] --- # TiDB Data Migration Glossary @@ -12,11 +11,11 @@ This document lists the terms used in the logs, monitoring, configurations, and ### Binlog -In TiDB DM, binlogs refer to the binary log files generated in the TiDB database. It has the same indications as that in MySQL or MariaDB. Refer to [MySQL Binary Log](https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_replication.html) and [MariaDB Binary Log](https://mariadb.com/kb/en/library/binary-log/) for details. +In TiDB DM, binlogs refer to the binary log files generated in the TiDB database. It has the same indications as that in MySQL or MariaDB. Refer to [MySQL Binary Log](https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_replication.html) and [MariaDB Binary Log](https://mariadb.com/docs/server/server-management/server-monitoring-logs/binary-log) for details. ### Binlog event -Binlog events are information about data modification made to a MySQL or MariaDB server instance. These binlog events are stored in the binlog files. Refer to [MySQL Binlog Event](https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_replication_binlog_event.html) and [MariaDB Binlog Event](https://mariadb.com/kb/en/library/1-binlog-events/) for details. +Binlog events are information about data modification made to a MySQL or MariaDB server instance. These binlog events are stored in the binlog files. Refer to [MySQL Binlog Event](https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_replication_binlog_event.html) and [MariaDB Binlog Event](https://mariadb.com/docs/server/reference/clientserver-protocol/replication-protocol/1-binlog-events) for details. ### Binlog event filter @@ -24,7 +23,7 @@ Binlog events are information about data modification made to a MySQL or MariaDB ### Binlog position -The binlog position is the offset information of a binlog event in a binlog file. Refer to [MySQL `SHOW BINLOG EVENTS`](https://dev.mysql.com/doc/refman/8.0/en/show-binlog-events.html) and [MariaDB `SHOW BINLOG EVENTS`](https://mariadb.com/kb/en/library/show-binlog-events/) for details. +The binlog position is the offset information of a binlog event in a binlog file. Refer to [MySQL `SHOW BINLOG EVENTS`](https://dev.mysql.com/doc/refman/8.0/en/show-binlog-events.html) and [MariaDB `SHOW BINLOG EVENTS`](https://mariadb.com/docs/server/reference/sql-statements/administrative-sql-statements/show/show-binlog-events) for details. ### Binlog replication processing unit/sync unit @@ -32,7 +31,7 @@ Binlog replication processing unit is the processing unit used in DM-worker to r ### Block & allow table list -Block & allow table list is the feature that filters or only migrates all operations of some databases or some tables. Refer to [block & allow table lists](/dm/dm-block-allow-table-lists.md) for details. This feature is similar to [MySQL Replication Filtering](https://dev.mysql.com/doc/refman/8.0/en/replication-rules.html) and [MariaDB Replication Filters](https://mariadb.com/kb/en/replication-filters/). +Block & allow table list is the feature that filters or only migrates all operations of some databases or some tables. Refer to [block & allow table lists](/dm/dm-block-allow-table-lists.md) for details. This feature is similar to [MySQL Replication Filtering](https://dev.mysql.com/doc/refman/8.0/en/replication-rules.html) and [MariaDB Replication Filters](https://mariadb.com/docs/server/ha-and-performance/standard-replication/replication-filters). ## C @@ -55,7 +54,7 @@ The dump processing unit is the processing unit used in DM-worker to export all ### GTID -The GTID is the global transaction ID of MySQL or MariaDB. With this feature enabled, the GTID information is recorded in the binlog files. Multiple GTIDs form a GTID set. Refer to [MySQL GTID Format and Storage](https://dev.mysql.com/doc/refman/8.0/en/replication-gtids-concepts.html) and [MariaDB Global Transaction ID](https://mariadb.com/kb/en/library/gtid/) for details. +The GTID is the global transaction ID of MySQL or MariaDB. With this feature enabled, the GTID information is recorded in the binlog files. Multiple GTIDs form a GTID set. Refer to [MySQL GTID Format and Storage](https://dev.mysql.com/doc/refman/8.0/en/replication-gtids-concepts.html) and [MariaDB Global Transaction ID](https://mariadb.com/docs/server/ha-and-performance/standard-replication/gtid) for details. ## L @@ -75,7 +74,7 @@ In the case of clearly mentioning "full", not explicitly mentioning "full or inc ### Relay log -The relay log refers to the binlog files that DM-worker pulls from the upstream MySQL or MariaDB, and stores in the local disk. The format of the relay log is the standard binlog file, which can be parsed by tools such as [mysqlbinlog](https://dev.mysql.com/doc/refman/8.0/en/mysqlbinlog.html) of a compatible version. Its role is similar to [MySQL Relay Log](https://dev.mysql.com/doc/refman/8.0/en/replica-logs-relaylog.html) and [MariaDB Relay Log](https://mariadb.com/kb/en/library/relay-log/). +The relay log refers to the binlog files that DM-worker pulls from the upstream MySQL or MariaDB, and stores in the local disk. The format of the relay log is the standard binlog file, which can be parsed by tools such as [mysqlbinlog](https://dev.mysql.com/doc/refman/8.0/en/mysqlbinlog.html) of a compatible version. Its role is similar to [MySQL Relay Log](https://dev.mysql.com/doc/refman/8.0/en/replica-logs-relaylog.html) and [MariaDB Relay Log](https://mariadb.com/docs/server/server-management/server-monitoring-logs/binary-log/relay-log). For more details such as the relay log's directory structure, initial migration rules, and data purge in TiDB DM, see [TiDB DM relay log](/dm/relay-log.md). diff --git a/dm/dm-hardware-and-software-requirements.md b/dm/dm-hardware-and-software-requirements.md index eb384aa2ee6e9..e4f6077305e14 100644 --- a/dm/dm-hardware-and-software-requirements.md +++ b/dm/dm-hardware-and-software-requirements.md @@ -1,7 +1,6 @@ --- title: Software and Hardware Requirements for TiDB Data Migration summary: Learn the software and hardware requirements for DM cluster. -aliases: ['/docs/tidb-data-migration/dev/hardware-and-software-requirements/'] --- # Software and Hardware Requirements for TiDB Data Migration @@ -55,20 +54,33 @@ The target TiKV cluster must have enough disk space to store the imported data. - Indexes might take extra space. - RocksDB has a space amplification effect. -You can estimate the data volume by using the following SQL statements to summarize the `data-length` field: - -- Calculate the size of all schemas, in MiB. Replace `${schema_name}` with your schema name. - - {{< copyable "sql" >}} - - ```sql - select table_schema,sum(data_length)/1024/1024 as data_length,sum(index_length)/1024/1024 as index_length,sum(data_length+index_length)/1024/1024 as sum from information_schema.tables where table_schema = "${schema_name}" group by table_schema; - ``` - -- Calculate the size of the largest table, in MiB. Replace ${schema_name} with your schema name. - - {{< copyable "sql" >}} - - ```sql - select table_name,table_schema,sum(data_length)/1024/1024 as data_length,sum(index_length)/1024/1024 as index_length,sum(data_length+index_length)/1024/1024 as sum from information_schema.tables where table_schema = "${schema_name}" group by table_name,table_schema order by sum desc limit 5; - ``` \ No newline at end of file +You can estimate the data volume by using the following SQL statements to summarize the `DATA_LENGTH` field: + +```sql +-- Calculate the size of all schemas +SELECT + TABLE_SCHEMA, + FORMAT_BYTES(SUM(DATA_LENGTH)) AS 'Data Size', + FORMAT_BYTES(SUM(INDEX_LENGTH)) 'Index Size' +FROM + information_schema.tables +GROUP BY + TABLE_SCHEMA; + +-- Calculate the 5 largest tables +SELECT + TABLE_NAME, + TABLE_SCHEMA, + FORMAT_BYTES(SUM(data_length)) AS 'Data Size', + FORMAT_BYTES(SUM(index_length)) AS 'Index Size', + FORMAT_BYTES(SUM(data_length+index_length)) AS 'Total Size' +FROM + information_schema.tables +GROUP BY + TABLE_NAME, + TABLE_SCHEMA +ORDER BY + SUM(DATA_LENGTH+INDEX_LENGTH) DESC +LIMIT + 5; +``` \ No newline at end of file diff --git a/dm/dm-master-configuration-file.md b/dm/dm-master-configuration-file.md index d32617c5b591f..de27887fec02e 100644 --- a/dm/dm-master-configuration-file.md +++ b/dm/dm-master-configuration-file.md @@ -1,7 +1,6 @@ --- title: DM-master Configuration File summary: Learn the configuration file of DM-master. -aliases: ['/docs/tidb-data-migration/dev/dm-master-configuration-file/'] --- # DM-master Configuration File diff --git a/dm/dm-open-api.md b/dm/dm-open-api.md index fc507018dee64..9ecbd257f4115 100644 --- a/dm/dm-open-api.md +++ b/dm/dm-open-api.md @@ -25,7 +25,7 @@ To enable OpenAPI, perform one of the following operations: > **Note:** > -> - DM provides the [specification document](https://github.com/pingcap/tiflow/blob/master/dm/openapi/spec/dm.yaml) that meets the OpenAPI 3.0.0 standard. This document contains all the request parameters and returned values. You can copy the document yaml and preview it in [Swagger Editor](https://editor.swagger.io/). +> - DM provides the [specification document](https://github.com/pingcap/tiflow/blob/release-7.5/dm/openapi/spec/dm.yaml) that meets the OpenAPI 3.0.0 standard. This document contains all the request parameters and returned values. You can copy the document yaml and preview it in [Swagger Editor](https://editor.swagger.io/). > > - After you deploy the DM-master nodes, you can access `http://{master-addr}/api/v1/docs` to preview the documentation online. > @@ -802,6 +802,16 @@ curl -X 'POST' \ "import_threads": 16, "data_dir": "./exported_data", "consistency": "auto" + "import_mode": "physical", + "sorting_dir": "./sort_dir", + "disk_quota": "80G", + "checksum": "required", + "analyze": "optional", + "range_concurrency": 0, + "compress-kv-pairs": "", + "pd_addr": "", + "on_duplicate_logical": "error", + "on_duplicate_physical": "none" }, "incr_migrate_conf": { "repl_threads": 16, @@ -892,6 +902,16 @@ curl -X 'POST' \ "import_threads": 16, "data_dir": "./exported_data", "consistency": "auto" + "import_mode": "physical", + "sorting_dir": "./sort_dir", + "disk_quota": "80G", + "checksum": "required", + "analyze": "optional", + "range_concurrency": 0, + "compress-kv-pairs": "", + "pd_addr": "", + "on_duplicate_logical": "error", + "on_duplicate_physical": "none" }, "incr_migrate_conf": { "repl_threads": 16, @@ -998,7 +1018,17 @@ curl -X 'GET' \ "export_threads": 4, "import_threads": 16, "data_dir": "./exported_data", - "consistency": "auto" + "consistency": "auto", + "import_mode": "physical", + "sorting_dir": "./sort_dir", + "disk_quota": "80G", + "checksum": "required", + "analyze": "optional", + "range_concurrency": 0, + "compress-kv-pairs": "", + "pd_addr": "", + "on_duplicate_logical": "error", + "on_duplicate_physical": "none" }, "incr_migrate_conf": { "repl_threads": 16, @@ -1126,7 +1156,17 @@ curl -X 'PUT' \ "export_threads": 4, "import_threads": 16, "data_dir": "./exported_data", - "consistency": "auto" + "consistency": "auto", + "import_mode": "physical", + "sorting_dir": "./sort_dir", + "disk_quota": "80G", + "checksum": "required", + "analyze": "optional", + "range_concurrency": 0, + "compress-kv-pairs": "", + "pd_addr": "", + "on_duplicate_logical": "error", + "on_duplicate_physical": "none" }, "incr_migrate_conf": { "repl_threads": 16, @@ -1217,6 +1257,16 @@ curl -X 'PUT' \ "import_threads": 16, "data_dir": "./exported_data", "consistency": "auto" + "import_mode": "physical", + "sorting_dir": "./sort_dir", + "disk_quota": "80G", + "checksum": "required", + "analyze": "optional", + "range_concurrency": 0, + "compress-kv-pairs": "", + "pd_addr": "", + "on_duplicate_logical": "error", + "on_duplicate_physical": "none" }, "incr_migrate_conf": { "repl_threads": 16, @@ -1296,7 +1346,7 @@ curl -X 'GET' \ "name": "string", "source_name": "string", "worker_name": "string", - "stage": "runing", + "stage": "running", "unit": "sync", "unresolved_ddl_lock_id": "string", "load_status": { @@ -1433,7 +1483,17 @@ curl -X 'GET' \ "export_threads": 4, "import_threads": 16, "data_dir": "./exported_data", - "consistency": "auto" + "consistency": "auto", + "import_mode": "physical", + "sorting_dir": "./sort_dir", + "disk_quota": "80G", + "checksum": "required", + "analyze": "optional", + "range_concurrency": 0, + "compress-kv-pairs": "", + "pd_addr": "", + "on_duplicate_logical": "error", + "on_duplicate_physical": "none" }, "incr_migrate_conf": { "repl_threads": 16, diff --git a/dm/dm-overview.md b/dm/dm-overview.md index 62726e1d3f1e4..40bb1839b50b3 100644 --- a/dm/dm-overview.md +++ b/dm/dm-overview.md @@ -1,7 +1,6 @@ --- title: TiDB Data Migration Overview summary: Learn about the Data Migration tool, the architecture, the key components, and features. -aliases: ['/docs/tidb-data-migration/dev/overview/','/docs/tidb-data-migration/dev/feature-overview/','/tidb/dev/dm-key-features'] --- @@ -12,11 +11,11 @@ aliases: ['/docs/tidb-data-migration/dev/overview/','/docs/tidb-data-migration/d ![star](https://img.shields.io/github/stars/pingcap/tiflow?style=for-the-badge&logo=github) ![license](https://img.shields.io/github/license/pingcap/tiflow?style=for-the-badge) ![forks](https://img.shields.io/github/forks/pingcap/tiflow?style=for-the-badge) --> -[TiDB Data Migration](https://github.com/pingcap/tiflow/tree/master/dm) (DM) is an integrated data migration task management platform, which supports the full data migration and the incremental data replication from MySQL-compatible databases (such as MySQL, MariaDB, and Aurora MySQL) into TiDB. It can help to reduce the operation cost of data migration and simplify the troubleshooting process. +[TiDB Data Migration](https://github.com/pingcap/tiflow/tree/release-7.5/dm) (DM) is an integrated data migration task management platform, which supports the full data migration and the incremental data replication from MySQL-compatible databases (such as MySQL, MariaDB, and Aurora MySQL) into TiDB. It can help to reduce the operation cost of data migration and simplify the troubleshooting process. ## Basic features -- **Compatibility with MySQL.** DM is compatible with MySQL 5.7 protocols and most of the features and syntax of MySQL 5.7. +- **Compatibility with MySQL.** DM is compatible with the MySQL protocol and most of the features and syntax of MySQL 5.7 and MySQL 8.0. - **Replicating DML and DDL events.** It supports parsing and replicating DML and DDL events in MySQL binlog. - **Migrating and merging MySQL shards.** DM supports migrating and merging multiple MySQL database instances upstream to one TiDB database downstream. It supports customizing replication rules for different migration scenarios. It can automatically detect and handle DDL changes of upstream MySQL shards, which greatly reduces the operational cost. - **Various types of filters.** You can predefine event types, regular expressions, and SQL expressions to filter out MySQL binlog events during the data migration process. @@ -66,15 +65,15 @@ Before using the DM tool, note the following restrictions: ## Contributing -You are welcome to participate in the DM open sourcing project. Your contribution would be highly appreciated. For more details, see [CONTRIBUTING.md](https://github.com/pingcap/tiflow/blob/master/dm/CONTRIBUTING.md). +You are welcome to participate in the DM open sourcing project. Your contribution would be highly appreciated. For more details, see [CONTRIBUTING.md](https://github.com/pingcap/tiflow/blob/release-7.5/dm/CONTRIBUTING.md). ## Community support -You can learn about DM through the online documentation. If you have any questions, contact us on [GitHub](https://github.com/pingcap/tiflow/tree/master/dm). +You can learn about DM through the online documentation. If you have any questions, contact us on [GitHub](https://github.com/pingcap/tiflow/tree/release-7.5/dm). ## License -DM complies with the Apache 2.0 license. For more details, see [LICENSE](https://github.com/pingcap/tiflow/blob/master/LICENSE). +DM complies with the Apache 2.0 license. For more details, see [LICENSE](https://github.com/pingcap/tiflow/blob/release-7.5/LICENSE). ## DM versions @@ -86,5 +85,5 @@ Before v5.4, the DM documentation is independent of the TiDB documentation. To a > **Note:** > -> - Since October 2021, DM's GitHub repository has been moved to [pingcap/tiflow](https://github.com/pingcap/tiflow/tree/master/dm). If you see any issues with DM, submit your issue to the `pingcap/tiflow` repository for feedback. -> - In earlier versions (v1.0 and v2.0), DM uses version numbers that are independent of TiDB. Since v5.3, DM uses the same version number as TiDB. The next version of DM v2.0 is DM v5.3. There are no compatibility changes from DM v2.0 to v5.3, and the upgrade process is the same as a normal upgrade, only an increase in version number. \ No newline at end of file +> - Since October 2021, DM's GitHub repository has been moved to [pingcap/tiflow](https://github.com/pingcap/tiflow/tree/release-7.5/dm). If you see any issues with DM, submit your issue to the `pingcap/tiflow` repository for feedback. +> - In earlier versions (v1.0 and v2.0), DM uses version numbers that are independent of TiDB. Since v5.3, DM uses the same version number as TiDB. The next version of DM v2.0 is DM v5.3. There are no compatibility changes from DM v2.0 to v5.3, and the upgrade process is the same as a normal upgrade, only an increase in version number. diff --git a/dm/dm-precheck.md b/dm/dm-precheck.md index 387111155d534..f1be8139026a4 100644 --- a/dm/dm-precheck.md +++ b/dm/dm-precheck.md @@ -1,7 +1,6 @@ --- title: Migration Task Precheck summary: Learn the precheck that DM performs before starting a migration task. -aliases: ['/docs/tidb-data-migration/dev/precheck/'] --- # Migration Task Precheck @@ -82,7 +81,7 @@ For the full data migration mode (`task-mode: full`), in addition to the [common - Primary key - Unique index - - In the optimistic mode, check whether the schemas of all sharded tables meet the [optimistic compatibility](https://github.com/pingcap/tiflow/blob/master/dm/docs/RFCS/20191209_optimistic_ddl.md#modifying-column-types). + - In the optimistic mode, check whether the schemas of all sharded tables meet the [optimistic compatibility](https://github.com/pingcap/tiflow/blob/release-7.5/dm/docs/RFCS/20191209_optimistic_ddl.md#modifying-column-types). - If a migration task was started successfully by the `start-task` command, the precheck of this task skips the consistency check. diff --git a/dm/dm-query-status.md b/dm/dm-query-status.md index 5a44caf4b4753..895e163aaa277 100644 --- a/dm/dm-query-status.md +++ b/dm/dm-query-status.md @@ -1,7 +1,6 @@ --- title: Query Task Status in TiDB Data Migration summary: Learn how to query the status of a data replication task. -aliases: ['/docs/tidb-data-migration/dev/query-status/'] --- # Query Task Status in TiDB Data Migration @@ -10,21 +9,26 @@ This document introduces how to use the `query-status` command to query the task ## Query result -{{< copyable "" >}} +It is recommended that you use `query-status` by the following steps: + +1. Use `query-status` to check whether each on-going task is in the normal state. +2. If any error occurs in a task, use the `query-status ` command to see detailed error information. `` in this command indicates the name of the task that encounters the error. + +A successful query result is as follows: ```bash » query-status ``` -``` +```json { - "result": true, # Whether the query is successful. - "msg": "", # Describes the reason for the unsuccessful query. - "tasks": [ # Migration task list. + "result": true, + "msg": "", + "tasks": [ { - "taskName": "test", # The task name. - "taskStatus": "Running", # The status of the task. - "sources": [ # The upstream MySQL list. + "taskName": "test", + "taskStatus": "Running", + "sources": [ "mysql-replica-01", "mysql-replica-02" ] @@ -41,12 +45,14 @@ This document introduces how to use the `query-status` command to query the task } ``` -For detailed descriptions of `taskStatus` under the `tasks` section, refer to [Task status](#task-status). - -It is recommended that you use `query-status` by the following steps: +Some fields in the query result are described as follows: -1. Use `query-status` to check whether each on-going task is in the normal state. -2. If any error occurs in a task, use the `query-status ` command to see detailed error information. `` in this command indicates the name of the task that encounters the error. +- `result`: Whether the query is successful. +- `msg`: The error message returned when the query fails. +- `tasks`: The list of migration tasks. Each task contains the following fields: + - `taskName`: The name of the task. + - `taskStatus`: The status of the task. For detailed descriptions of `taskStatus`, refer to [Task status](#task-status). + - `sources`: The list of upstream MySQL databases. ## Task status @@ -64,70 +70,59 @@ The status of a DM migration task depends on the status of each subtask assigned ## Detailed query result -{{< copyable "" >}} - ```bash » query-status test ``` -``` -» query-status +```json { - "result": true, # Whether the query is successful. - "msg": "", # Describes the cause for the unsuccessful query. - "sources": [ # The upstream MySQL list. + "result": true, + "msg": "", + "sources": [ { "result": true, "msg": "", - "sourceStatus": { # The information of the upstream MySQL databases. + "sourceStatus": { "source": "mysql-replica-01", "worker": "worker1", "result": null, "relayStatus": null }, - "subTaskStatus": [ # The information of all subtasks of upstream MySQL databases. + "subTaskStatus": [ { - "name": "test", # The name of the subtask. - "stage": "Running", # The running status of the subtask, including "New", "Running", "Paused", "Stopped", and "Finished". - "unit": "Sync", # The processing unit of DM, including "Check", "Dump", "Load", and "Sync". - "result": null, # Displays the error information if a subtask fails. - "unresolvedDDLLockID": "test-`test`.`t_target`", # The sharding DDL lock ID, used for manually handling the sharding DDL - # lock in the abnormal condition. - "sync": { # The replication information of the `Sync` processing unit. This information is about the - # same component with the current processing unit. - "masterBinlog": "(bin.000001, 3234)", # The binlog position in the upstream database. - "masterBinlogGtid": "c0149e17-dff1-11e8-b6a8-0242ac110004:1-14", # The GTID information in the upstream database. - "syncerBinlog": "(bin.000001, 2525)", # The position of the binlog that has been replicated - # in the `Sync` processing unit. - "syncerBinlogGtid": "", # The binlog position replicated using GTID. - "blockingDDLs": [ # The DDL list that is blocked currently. It is not empty only when all the upstream tables of this - # DM-worker are in the "synced" status. In this case, it indicates the sharding DDL statements to be executed or that are skipped. + "name": "test", + "stage": "Running", + "unit": "Sync", + "result": null, + "unresolvedDDLLockID": "test-`test`.`t_target`", + "sync": { + "masterBinlog": "(bin.000001, 3234)", + "masterBinlogGtid": "c0149e17-dff1-11e8-b6a8-0242ac110004:1-14", + "syncerBinlog": "(bin.000001, 2525)", + "syncerBinlogGtid": "", + "blockingDDLs": [ "USE `test`; ALTER TABLE `test`.`t_target` DROP COLUMN `age`;" ], - "unresolvedGroups": [ # The sharding group that is not resolved. + "unresolvedGroups": [ { - "target": "`test`.`t_target`", # The downstream database table to be replicated. + "target": "`test`.`t_target`", "DDLs": [ "USE `test`; ALTER TABLE `test`.`t_target` DROP COLUMN `age`;" ], - "firstPos": "(bin|000001.000001, 3130)", # The starting position of the sharding DDL statement. - "synced": [ # The upstream sharded table whose executed sharding DDL statement has been read by the `Sync` unit. + "firstPos": "(bin|000001.000001, 3130)", + "synced": [ "`test`.`t2`" "`test`.`t3`" "`test`.`t1`" ], - "unsynced": [ # The upstream table that has not executed this sharding DDL - # statement. If any upstream tables have not finished replication, - # `blockingDDLs` is empty. + "unsynced": [ ] } ], - "synced": false # Whether the incremental replication catches up with the upstream and has the same binlog position as that in the - # upstream. The save point is not refreshed in real time in the `Sync` background, so `false` of `synced` - # does not always mean a replication delay exits. - "totalRows": "12", # The total number of rows that are replicated in this subtask. - "totalRps": "1", # The number of rows that are replicated in this subtask per second. - "recentRps": "1" # The number of rows that are replicated in this subtask in the last second. + "synced": false, + "totalRows": "12", + "totalRps": "1", + "recentRps": "1" } } ] @@ -148,11 +143,11 @@ The status of a DM migration task depends on the status of each subtask assigned "unit": "Load", "result": null, "unresolvedDDLLockID": "", - "load": { # The replication information of the `Load` processing unit. - "finishedBytes": "115", # The number of bytes that have been loaded. - "totalBytes": "452", # The total number of bytes that need to be loaded. - "progress": "25.44 %", # The progress of the loading process. - "bps": "2734" # The speed of the full loading. + "load": { + "finishedBytes": "115", + "totalBytes": "452", + "progress": "25.44 %", + "bps": "2734" } } ] @@ -170,7 +165,7 @@ The status of a DM migration task depends on the status of each subtask assigned "name": "test", "stage": "Paused", "unit": "Load", - "result": { # The error example. + "result": { "isCanceled": false, "errors": [ { @@ -206,26 +201,65 @@ The status of a DM migration task depends on the status of each subtask assigned "unit": "Dump", "result": null, "unresolvedDDLLockID": "", - "dump": { # The replication information of the `Dump` processing unit. - "totalTables": "10", # The number of tables to be dumped. - "completedTables": "3", # The number of tables that have been dumped. - "finishedBytes": "2542", # The number of bytes that have been dumped. - "finishedRows": "32", # The number of rows that have been dumped. - "estimateTotalRows": "563", # The estimated number of rows to be dumped. - "progress": "30.52 %", # The progress of the dumping process. - "bps": "445" # The dumping speed. + "dump": { + "totalTables": "10", + "completedTables": "3", + "finishedBytes": "2542", + "finishedRows": "32", + "estimateTotalRows": "563", + "progress": "30.52 %", + "bps": "445" } } ] }, ] } - ``` -For the status description and status switch relationship of "stage" of "subTaskStatus" of "sources", see the [subtask status](#subtask-status). - -For operation details of "unresolvedDDLLockID" of "subTaskStatus" of "sources", see [Handle Sharding DDL Locks Manually](/dm/manually-handling-sharding-ddl-locks.md). +Some fields in the returned result are described as follows: + +- `result`: Whether the query is successful. +- `msg`: The error message returned when the query fails. +- `sources`: The list of upstream MySQL instances. Each source contains the following fields: + - `result` + - `msg` + - `sourceStatus`: The information of the upstream MySQL databases. + - `subTaskStatus`: The information of all subtasks of upstream MySQL databases. Each subtask might contain the following fields: + - `name`: The name of the subtask. + - `stage`: The status of the subtask. For the status description and status switch relationship of "stage" of "subTaskStatus" of "sources", see the [subtask status](#subtask-status). + - `unit`: The processing unit of DM, including "Check", "Dump", "Load", and "Sync". + - `result`: Displays the error information if a subtask fails. + - `unresolvedDDLLockID`: The sharding DDL lock ID, used for manually handling the sharding DDL lock in the abnormal condition. For operation details of "unresolvedDDLLockID" of "subTaskStatus" of "sources", see [Handle Sharding DDL Locks Manually](/dm/manually-handling-sharding-ddl-locks.md). + - `sync`: The replication information of the `Sync` processing unit. This information is about the same component with the current processing unit. + - `masterBinlog`: The binlog position in the upstream database. + - `masterBinlogGtid`: The GTID information in the upstream database. + - `syncerBinlog`: The position of the binlog that has been replicated in the `Sync` processing unit. + - `syncerBinlogGtid`: The binlog position replicated using GTID. + - `blockingDDLs`: The DDL list that is blocked currently. It is not empty only when all the upstream tables of this DM-worker are in the "synced" status. In this case, it indicates the sharding DDL statements to be executed or that are skipped. + - `unresolvedGroups`: The sharding group that is not resolved. Each group contains the following fields: + - `target`: The downstream database table to be replicated. + - `DDLs`: A list of DDL statements. + - `firstPos`: The starting position of the sharding DDL statement. + - `synced`: The upstream sharded table whose executed sharding DDL statement has been read by the `Sync` unit. + - `unsynced`: The upstream table that has not executed this sharding DDL statement. If any upstream tables have not finished replication, `blockingDDLs` is empty. + - `synced`: Whether the incremental replication catches up with the upstream and has the same binlog position as that in the upstream. The save point is not refreshed in real time in the `Sync` background, so `false` of `synced` does not always mean a replication delay exits. + - `totalRows`: The total number of rows that are replicated in this subtask. + - `totalRps`: The number of rows that are replicated in this subtask per second. + - `recentRps`: The number of rows that are replicated in this subtask in the last second. + - `load`: The replication information of the `Load` processing unit. + - `finishedBytes`: The number of bytes that have been loaded. + - `totalBytes`: The total number of bytes that need to be loaded. + - `progress`: The progress of the loading process. + - `bps`: The speed of the full loading. + - `dump`: The replication information of the `Dump` processing unit. + - `totalTables`: The number of tables to be dumped. + - `completedTables`: The number of tables that have been dumped. + - `finishedBytes`: The number of bytes that have been dumped. + - `finishedRows`: The number of rows that have been dumped. + - `estimateTotalRows`: The estimated number of rows to be dumped. + - `progress`: The progress of the dumping process. + - `bps`: The dumping speed in bytes/second. ## Subtask status diff --git a/dm/dm-release-notes.md b/dm/dm-release-notes.md index 888c94f050660..2946688181928 100644 --- a/dm/dm-release-notes.md +++ b/dm/dm-release-notes.md @@ -1,5 +1,6 @@ --- title: TiDB Data Migration Release Notes +summary: TiDB Data Migration Release Notes have been merged into TiDB Release Notes since DM v5.4. For DM Release Notes of v5.4 or later, see the corresponding TiDB Release Notes. For DM Release Notes of v5.3.0 or earlier, refer to the provided links for versions 5.3, 2.0, and 1.0. --- # TiDB Data Migration Release Notes diff --git a/dm/dm-source-configuration-file.md b/dm/dm-source-configuration-file.md index d540958e1e0d3..a38bebfef82a2 100644 --- a/dm/dm-source-configuration-file.md +++ b/dm/dm-source-configuration-file.md @@ -1,7 +1,6 @@ --- title: Upstream Database Configuration File of TiDB Data Migration summary: Learn the configuration file of the upstream database -aliases: ['/docs/tidb-data-migration/dev/source-configuration-file/'] --- # Upstream Database Configuration File of TiDB Data Migration @@ -19,7 +18,7 @@ source-id: "mysql-replica-01" enable-gtid: false # Whether to enable relay log. -enable-relay: false # Since DM v2.0.2, this configuration item is deprecated. To enable the relay log feature, use the `start-relay` command instead. +enable-relay: false relay-binlog-name: "" # The file name from which DM-worker starts to pull the binlog. relay-binlog-gtid: "" # The GTID from which DM-worker starts to pull the binlog. # relay-dir: "relay-dir" # The directory used to store relay log. The default value is "relay-dir". This configuration item is marked as deprecated since v6.1 and replaced by a parameter of the same name in the dm-worker configuration. @@ -70,7 +69,7 @@ This section describes each configuration parameter in the configuration file. | :------------ | :--------------------------------------- | | `source-id` | Represents a MySQL instance ID. | | `enable-gtid` | Determines whether to pull binlog from the upstream using GTID. The default value is `false`. In general, you do not need to configure `enable-gtid` manually. However, if GTID is enabled in the upstream database, and the primary/secondary switch is required, you need to set `enable-gtid` to `true`. | -| `enable-relay` | Determines whether to enable the relay log feature. The default value is `false`. Since DM v2.0.2, this configuration item is deprecated. To [enable the relay log feature](/dm/relay-log.md#enable-and-disable-relay-log), use the `start-relay` command instead. | +| `enable-relay` | Determines whether to enable the relay log feature. The default value is `false`. This parameter takes effect from v5.4. Additionally, you can [enable relay log dynamically](/dm/relay-log.md#enable-and-disable-relay-log) using the `start-relay` command. | | `relay-binlog-name` | Specifies the file name from which DM-worker starts to pull the binlog. For example, `"mysql-bin.000002"`. It only works when `enable_gtid` is `false`. If this parameter is not specified, DM-worker will start pulling from the earliest binlog file being replicated. Manual configuration is generally not required. | | `relay-binlog-gtid` | Specifies the GTID from which DM-worker starts to pull the binlog. For example, `"e9a1fc22-ec08-11e9-b2ac-0242ac110003:1-7849"`. It only works when `enable_gtid` is `true`. If this parameter is not specified, DM-worker will start pulling from the latest GTID being replicated. Manual configuration is generally not required. | | `relay-dir` | Specifies the relay log directory. | diff --git a/dm/dm-stop-task.md b/dm/dm-stop-task.md index 680c6bd7ab6bb..c5c9c5ba212d7 100644 --- a/dm/dm-stop-task.md +++ b/dm/dm-stop-task.md @@ -61,4 +61,11 @@ stop-task test } ] } -``` \ No newline at end of file +``` + +> **Note:** +> +> After you stop a migration task with the `stop-task` command, running [`query-status`](/dm/dm-query-status.md) will no longer display the task. However, the checkpoint and other related information for this task is still retained in the `dm_meta` database. +> +> + To start over the migration task, add the `--remove-meta` option when running the [`start-task`](/dm/dm-create-task.md) command. +> + To completely remove the migration task, manually delete the four tables that use the task name as a prefix from the `dm_meta` database. \ No newline at end of file diff --git a/dm/dm-table-routing.md b/dm/dm-table-routing.md index 63f1089e56ef6..2625c6aaf1fb1 100644 --- a/dm/dm-table-routing.md +++ b/dm/dm-table-routing.md @@ -86,7 +86,7 @@ To migrate the upstream instances to the downstream `test`.`t`, you must create Assuming in the scenario of sharded schemas and tables, you want to migrate the `test_{1,2,3...}`.`t_{1,2,3...}` tables in two upstream MySQL instances to the `test`.`t` table in the downstream TiDB instance. At the same time, you want to extract the source information of the sharded tables and write it to the downstream merged table. -To migrate the upstream instances to the downstream `test`.`t`, you must create routing rules similar to the previous section [Merge sharded schemas and tables](#merge-sharded-schemas-and-tables). In addtion, you need to add the `extract-table`, `extract-schema`, and `extract-source` configurations: +To migrate the upstream instances to the downstream `test`.`t`, you must create routing rules similar to the previous section [Merge sharded schemas and tables](#merge-sharded-schemas-and-tables). In addition, you need to add the `extract-table`, `extract-schema`, and `extract-source` configurations: - `extract-table`: For a sharded table matching `schema-pattern` and `table-pattern`, DM extracts the sharded table name by using `table-regexp` and writes the name suffix without the `t_` part to `target-column` of the merged table, that is, the `c_table` column. - `extract-schema`: For a sharded schema matching `schema-pattern` and `table-pattern`, DM extracts the sharded schema name by using `schema-regexp` and writes the name suffix without the `test_` part to `target-column` of the merged table, that is, the `c_schema` column. diff --git a/dm/dm-worker-configuration-file.md b/dm/dm-worker-configuration-file.md index 39c3614ccc50d..fb437e26eff5f 100644 --- a/dm/dm-worker-configuration-file.md +++ b/dm/dm-worker-configuration-file.md @@ -1,7 +1,6 @@ --- title: DM-worker Configuration File summary: Learn the configuration file of DM-worker. -aliases: ['/docs/tidb-data-migration/dev/dm-worker-configuration-file/','/docs/tidb-data-migration/dev/dm-worker-configuration-file-full/'] --- # DM-worker Configuration File diff --git a/dm/dm-worker-intro.md b/dm/dm-worker-intro.md index c5c2c2c8ed9c4..a6328065ee005 100644 --- a/dm/dm-worker-intro.md +++ b/dm/dm-worker-intro.md @@ -1,7 +1,6 @@ --- title: DM-worker Introduction summary: Learn the features of DM-worker. -aliases: ['/docs/tidb-data-migration/dev/dm-worker-intro/'] --- # DM-worker Introduction @@ -55,7 +54,7 @@ The upstream database (MySQL/MariaDB) user must have the following privileges: If you need to migrate the data from `db1` to TiDB, execute the following `GRANT` statement: ```sql -GRANT RELOAD,REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'your_user'@'your_wildcard_of_host' +GRANT RELOAD,REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'your_user'@'your_wildcard_of_host'; GRANT SELECT ON db1.* TO 'your_user'@'your_wildcard_of_host'; ``` diff --git a/dm/dmctl-introduction.md b/dm/dmctl-introduction.md index a14f4598b1d76..1187651297a8b 100644 --- a/dm/dmctl-introduction.md +++ b/dm/dmctl-introduction.md @@ -1,7 +1,6 @@ --- title: Maintain DM Clusters Using dmctl summary: Learn how to maintain a DM cluster using dmctl. -aliases: ['/docs/tidb-data-migration/dev/manage-replication-tasks/'] --- # Maintain DM Clusters Using dmctl diff --git a/dm/feature-expression-filter.md b/dm/feature-expression-filter.md index f256e3db46606..e3ed93cee0085 100644 --- a/dm/feature-expression-filter.md +++ b/dm/feature-expression-filter.md @@ -1,6 +1,6 @@ --- title: Filter DMLs Using SQL Expressions -aliases: ['/tidb/dev/feature-expression-filter/'] +summary: In incremental data migration, you can filter binlog events using SQL expressions. DM supports filtering data during migration using binlog value filter since v2.0.5. You can configure SQL expressions based on the values in binlog events to determine whether to migrate a row change downstream. For detailed operation and implementation, refer to "Filter DML Events Using SQL Expressions". --- # Filter DMLs Using SQL Expressions diff --git a/dm/feature-online-ddl.md b/dm/feature-online-ddl.md index e853ef67ff688..121539fd62dd5 100644 --- a/dm/feature-online-ddl.md +++ b/dm/feature-online-ddl.md @@ -1,7 +1,6 @@ --- title: Migrate from Databases that Use GH-ost/PT-osc summary: This document introduces the `online-ddl/online-ddl-scheme` feature of DM. -aliases: ['/docs/tidb-data-migration/dev/online-ddl-scheme/','tidb-data-migration/dev/feature-online-ddl-scheme'] --- # Migrate from Databases that Use GH-ost/PT-osc diff --git a/dm/feature-shard-merge.md b/dm/feature-shard-merge.md index 7bc428738efab..2ebf9812dd19b 100644 --- a/dm/feature-shard-merge.md +++ b/dm/feature-shard-merge.md @@ -1,7 +1,6 @@ --- title: Merge and Migrate Data from Sharded Tables summary: Learn how DM merges and migrates data from sharded tables. -aliases: ['/docs/tidb-data-migration/dev/feature-shard-merge/'] --- # Merge and Migrate Data from Sharded Tables diff --git a/dm/handle-failed-ddl-statements.md b/dm/handle-failed-ddl-statements.md index 99415b02ee934..aef8aa90200f2 100644 --- a/dm/handle-failed-ddl-statements.md +++ b/dm/handle-failed-ddl-statements.md @@ -1,7 +1,6 @@ --- title: Handle Failed DDL Statements in TiDB Data Migration summary: Learn how to handle failed DDL statements when you're using the TiDB Data Migration tool to migrate data. -aliases: ['/docs/tidb-data-migration/dev/skip-or-replace-abnormal-sql-statements/'] --- # Handle Failed DDL Statements in TiDB Data Migration diff --git a/dm/maintain-dm-using-tiup.md b/dm/maintain-dm-using-tiup.md index e0461599ff4cf..488de529fd81f 100644 --- a/dm/maintain-dm-using-tiup.md +++ b/dm/maintain-dm-using-tiup.md @@ -1,7 +1,6 @@ --- title: Maintain a DM Cluster Using TiUP summary: Learn how to maintain a DM cluster using TiUP. -aliases: ['/docs/tidb-data-migration/dev/cluster-operations/'] --- # Maintain a DM Cluster Using TiUP @@ -390,7 +389,7 @@ All operations above performed on the cluster machine use the SSH client embedde Then you can use the `--native-ssh` command-line flag to enable the system-native command-line tool: -- Deploy a cluster: `tiup dm deploy --native-ssh`. Fill in the name of your cluster for ``, the DM version to be deployed (such as `v7.4.0`) for `` , and the topology file name for ``. +- Deploy a cluster: `tiup dm deploy --native-ssh`. Fill in the name of your cluster for ``, the DM version to be deployed (such as `v{{{ .tidb-version }}}`) for `` , and the topology file name for ``. - Start a cluster: `tiup dm start --native-ssh`. - Upgrade a cluster: `tiup dm upgrade ... --native-ssh` diff --git a/dm/manually-handling-sharding-ddl-locks.md b/dm/manually-handling-sharding-ddl-locks.md index 9780b5b8172fa..ccae25b79b77a 100644 --- a/dm/manually-handling-sharding-ddl-locks.md +++ b/dm/manually-handling-sharding-ddl-locks.md @@ -1,7 +1,6 @@ --- title: Handle Sharding DDL Locks Manually in DM summary: Learn how to handle sharding DDL locks manually in DM. -aliases: ['/docs/tidb-data-migration/dev/feature-manually-handling-sharding-ddl-locks/'] --- # Handle Sharding DDL Locks Manually in DM @@ -55,7 +54,7 @@ You can use `shard-ddl-lock [task] [flags]` to view the DDL lock information on shard-ddl-lock test ``` -
+
Expected output ``` diff --git a/dm/migrate-data-using-dm.md b/dm/migrate-data-using-dm.md index 395b03d2c3d3f..5cb316c28ee1a 100644 --- a/dm/migrate-data-using-dm.md +++ b/dm/migrate-data-using-dm.md @@ -1,7 +1,6 @@ --- title: Migrate Data Using Data Migration summary: Use the Data Migration tool to migrate the full data and the incremental data. -aliases: ['/docs/tidb-data-migration/dev/replicate-data-using-dm/'] --- # Migrate Data Using Data Migration diff --git a/dm/monitor-a-dm-cluster.md b/dm/monitor-a-dm-cluster.md index f40a66dc3d709..9c77d0e413194 100644 --- a/dm/monitor-a-dm-cluster.md +++ b/dm/monitor-a-dm-cluster.md @@ -1,7 +1,6 @@ --- title: Data Migration Monitoring Metrics summary: Learn about the monitoring metrics when you use Data Migration to migrate data. -aliases: ['/docs/tidb-data-migration/dev/monitor-a-dm-cluster/'] --- # Data Migration Monitoring Metrics @@ -94,7 +93,7 @@ The following metrics show only when `task-mode` is in the `incremental` or `all | total sqls jobs | The number of newly added jobs per unit of time | N/A | N/A | | finished sqls jobs | The number of finished jobs per unit of time | N/A | N/A | | statement execution latency | The duration that the binlog replication unit executes the statement to the downstream (in seconds) | N/A | N/A | -| add job duration | The duration tht the binlog replication unit adds a job to the queue (in seconds) | N/A | N/A | +| add job duration | The duration that the binlog replication unit adds a job to the queue (in seconds) | N/A | N/A | | DML conflict detect duration | The duration that the binlog replication unit detects the conflict in DML (in seconds) | N/A | N/A | | skipped event duration | The duration that the binlog replication unit skips a binlog event (in seconds) | N/A | N/A | | unsynced tables | The number of tables that have not received the shard DDL statement in the current subtask | N/A | N/A | diff --git a/dm/quick-start-create-source.md b/dm/quick-start-create-source.md index 23886f4eaf8b9..5013e9d147d26 100644 --- a/dm/quick-start-create-source.md +++ b/dm/quick-start-create-source.md @@ -82,7 +82,7 @@ The returned results are as follows: After creating a data source, you can use the following command to query the data source: -- If you konw the `source-id` of the data source, you can use the `dmctl config source ` command to directly check the configuration of the data source: +- If you know the `source-id` of the data source, you can use the `dmctl config source ` command to directly check the configuration of the data source: {{< copyable "shell-regular" >}} diff --git a/dm/quick-start-create-task.md b/dm/quick-start-create-task.md index 367fb1bf1b6e2..60359835baa15 100644 --- a/dm/quick-start-create-task.md +++ b/dm/quick-start-create-task.md @@ -1,7 +1,6 @@ --- title: Create a Data Migration Task summary: Learn how to create a migration task after the DM cluster is deployed. -aliases: ['/docs/tidb-data-migration/dev/create-task-and-verify/'] --- # Create a Data Migration Task @@ -74,7 +73,7 @@ To run a TiDB server, use the following command: {{< copyable "shell-regular" >}} ```bash -wget https://download.pingcap.org/tidb-community-server-v7.4.0-linux-amd64.tar.gz +wget https://download.pingcap.com/tidb-community-server-v{{{ .tidb-version }}}-linux-amd64.tar.gz tar -xzvf tidb-latest-linux-amd64.tar.gz mv tidb-latest-linux-amd64/bin/tidb-server ./ ./tidb-server diff --git a/dm/quick-start-with-dm.md b/dm/quick-start-with-dm.md index 75231181e152d..a4b7e55644407 100644 --- a/dm/quick-start-with-dm.md +++ b/dm/quick-start-with-dm.md @@ -1,12 +1,11 @@ --- title: TiDB Data Migration Quick Start summary: Learn how to quickly deploy a DM cluster using binary packages. -aliases: ['/docs/tidb-data-migration/dev/get-started/'] --- # Quick Start Guide for TiDB Data Migration -This document describes how to migrate data from MySQL to TiDB using [TiDB Data Migration](https://github.com/pingcap/dm) (DM). This guide is a quick demo of DM features and is not recommended for any production environment. +This document describes how to migrate data from MySQL to TiDB using [TiDB Data Migration (DM)](/dm/dm-overview.md). This guide is a quick demo of DM features and is not recommended for any production environment. ## Step 1: Deploy a DM cluster @@ -32,7 +31,7 @@ This document describes how to migrate data from MySQL to TiDB using [TiDB Data {{< copyable "shell-regular" >}} ```shell - tiup dm deploy dm-test 6.0.0 topology.yaml -p + tiup dm deploy dm-test {{{ .tidb-version }}} topology.yaml -p ``` ## Step 2: Prepare the data source diff --git a/dm/relay-log.md b/dm/relay-log.md index cafb2f51011d0..e630b7a26ada2 100644 --- a/dm/relay-log.md +++ b/dm/relay-log.md @@ -1,7 +1,6 @@ --- title: Data Migration Relay Log summary: Learn the directory structure, initial migration rules and data purge of DM relay logs. -aliases: ['/docs/tidb-data-migration/dev/relay-log/'] --- # Data Migration Relay Log diff --git a/dm/shard-merge-best-practices.md b/dm/shard-merge-best-practices.md index 507f276605019..7169d599e5358 100644 --- a/dm/shard-merge-best-practices.md +++ b/dm/shard-merge-best-practices.md @@ -1,12 +1,11 @@ --- title: Best Practices of Data Migration in the Shard Merge Scenario summary: Learn the best practices of data migration in the shard merge scenario. -aliases: ['/docs/tidb-data-migration/dev/shard-merge-best-practices/'] --- # Best Practices of Data Migration in the Shard Merge Scenario -This document describes the features and limitations of [TiDB Data Migration](https://github.com/pingcap/dm) (DM) in the shard merge scenario and provides a data migration best practice guide for your application (the default "pessimistic" mode is used). +This document describes the features and limitations of [TiDB Data Migration (DM)](/dm/dm-overview.md) in the shard merge scenario and provides a data migration best practice guide for your application (the default "pessimistic" mode is used). ## Use a separate data migration task diff --git a/dm/table-selector.md b/dm/table-selector.md index ee05ace71b3d8..b3eef57182ff6 100644 --- a/dm/table-selector.md +++ b/dm/table-selector.md @@ -1,7 +1,6 @@ --- title: Table Selector of TiDB Data Migration summary: Learn about Table Selector used by the table routing, binlog event filtering, and column mapping rule of Data Migration. -aliases: ['/docs/tidb-data-migration/dev/table-selector/'] --- # Table Selector of TiDB Data Migration diff --git a/dm/task-configuration-file-full.md b/dm/task-configuration-file-full.md index c1d81955a3467..7b8e725352bf2 100644 --- a/dm/task-configuration-file-full.md +++ b/dm/task-configuration-file-full.md @@ -1,6 +1,6 @@ --- title: DM Advanced Task Configuration File -aliases: ['/docs/tidb-data-migration/dev/task-configuration-file-full/','/docs/tidb-data-migration/dev/dm-portal/'] +summary: This document introduces the advanced task configuration file of Data Migration (DM), covering global and instance configuration. The global configuration includes basic and feature settings, while the instance configuration defines subtasks for data migration from one or multiple MySQL instances in the upstream to the same instance in the downstream. --- # DM Advanced Task Configuration File @@ -45,6 +45,7 @@ target-database: # Configuration of the downstream database insta sql_mode: "ANSI_QUOTES,NO_ZERO_IN_DATE,NO_ZERO_DATE" # Since DM v2.0.0, if this item does not appear in the configuration file, DM automatically fetches a proper value for "sql_mode" from the downstream TiDB. Manual configuration of this item has a higher priority. tidb_skip_utf8_check: 1 # Since DM v2.0.0, if this item does not appear in the configuration file, DM automatically fetches a proper value for "tidb_skip_utf8_check" from the downstream TiDB. Manual configuration of this item has a higher priority. tidb_constraint_check_in_place: 0 + sql_require_primary_key: OFF # Controls whether a table must have a primary key at the session level. During the creation of a DM task, several metadata tables are created in TiDB, and some of them have no primary key. If this parameter is enabled, these metadata tables without primary keys cannot be created, which will cause DM to fail to create the task. In this case, you need to set this parameter to `OFF`. security: # The TLS configuration of the downstream TiDB ssl-ca: "/path/to/ca.pem" ssl-cert: "/path/to/cert.pem" @@ -59,7 +60,7 @@ routes: table-pattern: "t_*" # The pattern of the upstream table name, wildcard characters (*?) are supported. target-schema: "test" # The name of the downstream schema. target-table: "t" # The name of the downstream table. - # Optional. Used for extracting the source information of sharded schemas and tables and writing the information to the user-defined columns in the downstream. If these options are configured, you need to manually create a merged table in the downstream. For details, see description of table routing in . + # Optional. Used for extracting the source information of sharded schemas and tables and writing the information to the user-defined columns in the downstream. If these options are configured, you need to manually create a merged table in the downstream. For details, see description of table routing in . # extract-table: # Extracts and writes the table name suffix without the t_ part to the c-table column of the merged table. For example, 01 is extracted and written to the c-table column for the sharded table t_01. # table-regexp: "t_(.*)" # target-column: "c_table" @@ -124,7 +125,7 @@ loaders: # The import mode during the full import phase. The following modes are supported: # - "logical" (default). Uses TiDB Lightning's logical import mode to import data. Document: https://docs.pingcap.com/tidb/stable/tidb-lightning-logical-import-mode - # - "physical". Uses TiDB Lightning's physical import mode to import data. Document: https://docs.pingcap.com/zh/tidb/stable/tidb-lightning-physical-import-mode + # - "physical". Uses TiDB Lightning's physical import mode to import data. Document: https://docs.pingcap.com/tidb/stable/tidb-lightning-physical-import-mode # The "physical" mode is still an experimental feature and is not recommended in production. import-mode: "logical" # Methods to resolve conflicts in logical import. diff --git a/download-ecosystem-tools.md b/download-ecosystem-tools.md index fe65a624f7cc1..74b15fe2cfafd 100644 --- a/download-ecosystem-tools.md +++ b/download-ecosystem-tools.md @@ -1,7 +1,6 @@ --- title: Download TiDB Tools summary: Download the most officially maintained versions of TiDB tools. -aliases: ['/docs/dev/download-ecosystem-tools/','/docs/dev/reference/tools/download/'] --- # Download TiDB Tools @@ -25,14 +24,14 @@ TiDB Toolkit contains frequently used TiDB tools, such as data export tool Dumpl You can download TiDB Toolkit from the following link: ``` -https://download.pingcap.org/tidb-community-toolkit-{version}-linux-{arch}.tar.gz +https://download.pingcap.com/tidb-community-toolkit-{version}-linux-{arch}.tar.gz ``` -`{version}` in the link indicates the version number of TiDB and `{arch}` indicates the architecture of the system, which can be `amd64` or `arm64`. For example, the download link for `v6.2.0` in the `amd64` architecture is `https://download.pingcap.org/tidb-community-toolkit-v6.2.0-linux-amd64.tar.gz`. +`{version}` in the link indicates the version number of TiDB and `{arch}` indicates the architecture of the system, which can be `amd64` or `arm64`. For example, the download link for `v6.2.0` in the `amd64` architecture is `https://download.pingcap.com/tidb-community-toolkit-v6.2.0-linux-amd64.tar.gz`. > **Note:** > -> If you need to download the [PD Control](/pd-control.md) tool `pd-ctl`, download the TiDB installation package separately from `https://download.pingcap.org/tidb-community-server-{version}-linux-{arch}.tar.gz`. +> If you need to download the [PD Control](/pd-control.md) tool `pd-ctl`, download the TiDB installation package separately from `https://download.pingcap.com/tidb-community-server-{version}-linux-{arch}.tar.gz`. ## TiDB Toolkit description diff --git a/dr-multi-replica.md b/dr-multi-replica.md index fb8d2ed065cd5..fa85092ede9fe 100644 --- a/dr-multi-replica.md +++ b/dr-multi-replica.md @@ -74,8 +74,8 @@ In this example, TiDB contains five replicas and three regions. Region 1 is the config: server.labels: { Region: "Region3", AZ: "AZ5" } - raftstore.raft-min-election-timeout-ticks: 1000 - raftstore.raft-max-election-timeout-ticks: 1200 + raftstore.raft-min-election-timeout-ticks: 50 + raftstore.raft-max-election-timeout-ticks: 60 monitoring_servers: - host: tidb-dr-test2 diff --git a/dr-secondary-cluster.md b/dr-secondary-cluster.md index d8160cb1794df..37126f1e3e58d 100644 --- a/dr-secondary-cluster.md +++ b/dr-secondary-cluster.md @@ -299,13 +299,20 @@ It is important to conduct regular DR drills for critical business systems to te 2. After there are no more writes, query the latest TSO (`Position`) of the TiDB cluster: ```sql - mysql> show master status; - +-------------+--------------------+--------------+------------------+-------------------+ - | File | Position | Binlog_Do_DB | Binlog_Ignore_DB | Executed_Gtid_Set | - +-------------+--------------------+--------------+------------------+-------------------+ - | tidb-binlog | 438223974697009153 | | | | - +-------------+--------------------+--------------+------------------+-------------------+ - 1 row in set (0.33 sec) + BEGIN; SELECT TIDB_CURRENT_TSO(); ROLLBACK; + ``` + + ```sql + Query OK, 0 rows affected (0.00 sec) + + +--------------------+ + | TIDB_CURRENT_TSO() | + +--------------------+ + | 452654700157468673 | + +--------------------+ + 1 row in set (0.00 sec) + + Query OK, 0 rows affected (0.00 sec) ``` 3. Poll the changefeed `dr-primary-to-secondary` until it meets the condition `TSO >= Position`. diff --git a/dr-solution-introduction.md b/dr-solution-introduction.md index ca3a80cc91e0c..dd70a150d0c78 100644 --- a/dr-solution-introduction.md +++ b/dr-solution-introduction.md @@ -93,7 +93,7 @@ Of course, if the error tolerance objective is multiple regions and RPO must be In this architecture, TiDB cluster 1 is deployed in region 1. BR regularly backs up the data of cluster 1 to region 2, and continuously backs up the data change logs of this cluster to region 2 as well. When region 1 encounters a disaster and cluster 1 cannot be recovered, you can use the backup data and data change logs to restore a new cluster (cluster 2) in region 2 to provide services. -The DR solution based on BR provides an RPO lower than 5 minutes and an RTO that varies with the size of the data to be restored. For BR v6.5.0, you can refer to [Performance and impact of snapshot restore](/br/br-snapshot-guide.md#performance-and-impact-of-snapshot-restore) and [Performance and impact of PITR](/br/br-pitr-guide.md#performance-and-impact-of-pitr) to learn about the restore speed. Usually, the feature of backup across regions is considered the last resort of data security and also a must-have solution for most systems. For more information about this solution, see [DR solution based on BR](/dr-backup-restore.md). +The DR solution based on BR provides an RPO lower than 5 minutes and an RTO that varies with the size of the data to be restored. For BR v6.5.0, you can refer to [Performance and impact of snapshot restore](/br/br-snapshot-guide.md#performance-and-impact-of-snapshot-restore) and [Performance and impact of PITR](/br/br-pitr-guide.md#performance-capabilities-of-pitr) to learn about the restore speed. Usually, the feature of backup across regions is considered the last resort of data security and also a must-have solution for most systems. For more information about this solution, see [DR solution based on BR](/dr-backup-restore.md). Meanwhile, starting from v6.5.0, BR supports [restoring a TiDB cluster from EBS volume snapshots](https://docs.pingcap.com/tidb-in-kubernetes/stable/restore-from-aws-s3-by-snapshot). If your cluster is running on Kubernetes and you want to restore the cluster as fast as possible without affecting the cluster, you can use this feature to reduce the RTO of your system. diff --git a/dumpling-overview.md b/dumpling-overview.md index bc40f54d704c2..f6d2bcda99418 100644 --- a/dumpling-overview.md +++ b/dumpling-overview.md @@ -1,12 +1,11 @@ --- title: Dumpling Overview summary: Use the Dumpling tool to export data from TiDB. -aliases: ['/docs/dev/mydumper-overview/','/docs/dev/reference/tools/mydumper/','/tidb/dev/mydumper-overview/'] --- # Use Dumpling to Export Data -This document introduces the data export tool - [Dumpling](https://github.com/pingcap/tidb/tree/master/dumpling). Dumpling exports data stored in TiDB/MySQL as SQL or CSV data files and can be used to make a logical full backup or export. Dumpling also supports exporting data to Amazon S3. +This document introduces the data export tool - [Dumpling](https://github.com/pingcap/tidb/tree/release-7.5/dumpling). Dumpling exports data stored in TiDB/MySQL as SQL or CSV data files and can be used to make a logical full backup or export. Dumpling also supports exporting data to Amazon S3. @@ -55,7 +54,7 @@ Dumpling has the following advantages: - Support exporting data to Amazon S3 cloud storage. - More optimizations are made for TiDB: - Support configuring the memory limit of a single TiDB SQL statement. - - If Dumpling can connect directly to PD, Dumpling supports automatic adjustment of TiDB GC time for TiDB v4.0.0 and later versions. + - If Dumpling can access the PD address and the [`INFORMATION_SCHEMA.CLUSTER_INFO`](/information-schema/information-schema-cluster-info.md) table of the TiDB cluster, Dumpling supports automatically adjusting the [GC](/garbage-collection-overview.md) safe point time to block GC for TiDB v4.0.0 and later versions. - Use TiDB's hidden column `_tidb_rowid` to optimize the performance of concurrent data export from a single table. - For TiDB, you can set the value of [`tidb_snapshot`](/read-historical-data.md#how-tidb-reads-data-from-history-versions) to specify the time point of the data backup. This ensures the consistency of the backup, instead of using `FLUSH TABLES WITH READ LOCK` to ensure the consistency. @@ -95,7 +94,7 @@ In the command above: + The `-h`, `-P`, and `-u` option respectively mean the address, the port, and the user. If a password is required for authentication, you can use `-p $YOUR_SECRET_PASSWORD` to pass the password to Dumpling. + The `-o` (or `--output`) option specifies the export directory of the storage, which supports an absolute local file path or an [external storage URI](/external-storage-uri.md). + The `-t` option specifies the number of threads for the export. Increasing the number of threads improves the concurrency of Dumpling and the export speed, and also increases the database's memory consumption. Therefore, it is not recommended to set the number too large. Usually, it's less than 64. -+ The `-r` option enables the in-table concurrency to speed up the export. The default value is `0`, which means disabled. A value greater than 0 means it is enabled, and the value is of `INT` type. When the source database is TiDB, a `-r` value greater than 0 indicates that the TiDB region information is used for splitting, and reduces the memory usage. The specific `-r` value does not affect the split algorithm. When the source database is MySQL and the primary key is of the `INT` type, specifying `-r` can also enable the in-table concurrency. ++ The `-r` option enables the in-table concurrency to speed up the export. The default value is `0`, which means disabled. A value greater than 0 means it is enabled, and the value is of `INT` type. When the source database is TiDB, a `-r` value greater than 0 indicates that the TiDB region information is used for splitting, and reduces the memory usage. The specific `-r` value does not affect the split algorithm. When the source database is MySQL and the primary key or the first column of the composite primary key is of the `INT` type, specifying `-r` can also enable the in-table concurrency. + The `-F` option is used to specify the maximum size of a single file (the unit here is `MiB`; inputs like `5GiB` or `8KB` are also acceptable). It is recommended to keep its value to 256 MiB or less if you plan to use TiDB Lightning to load this file into a TiDB instance. > **Note:** @@ -282,7 +281,7 @@ Examples: The exported file is stored in the `./export-` directory by default. Commonly used options are as follows: - The `-t` option specifies the number of threads for the export. Increasing the number of threads improves the concurrency of Dumpling and the export speed, and also increases the database's memory consumption. Therefore, it is not recommended to set the number too large. -- The `-r` option enables the in-table concurrency to speed up the export. The default value is `0`, which means disabled. A value greater than 0 means it is enabled, and the value is of `INT` type. When the source database is TiDB, a `-r` value greater than 0 indicates that the TiDB region information is used for splitting, and reduces the memory usage. The specific `-r` value does not affect the split algorithm. When the source database is MySQL and the primary key is of the `INT` type, specifying `-r` can also enable the in-table concurrency. +- The `-r` option enables the in-table concurrency to speed up the export. The default value is `0`, which means disabled. A value greater than 0 means it is enabled, and the value is of `INT` type. When the source database is TiDB, a `-r` value greater than 0 indicates that the TiDB region information is used for splitting, and reduces the memory usage. The specific `-r` value does not affect the split algorithm. When the source database is MySQL and the primary key or the first column of the composite primary key is of the `INT` type, specifying `-r` can also enable the in-table concurrency. - The `--compress ` option specifies the compression format of the dump. It supports the following compression algorithms: `gzip`, `snappy`, and `zstd`. This option can speed up dumping of data if storage is the bottleneck or if storage capacity is a concern. The drawback is an increase in CPU usage. Each file is compressed individually. With the above options specified, Dumpling can have a quicker speed of data export. @@ -345,7 +344,7 @@ When Dumpling is exporting a large single table from TiDB, Out of Memory (OOM) m ### Manually set the TiDB GC time -When exporting data from TiDB (more than 1 TB), if the TiDB version is later than or equal to v4.0.0 and Dumpling can access the PD address of the TiDB cluster, Dumpling automatically extends the GC time without affecting the original cluster. +When exporting data from TiDB (less than 1 TB), if the TiDB version is v4.0.0 or later and Dumpling can access the PD address and the [`INFORMATION_SCHEMA.CLUSTER_INFO`](/information-schema/information-schema-cluster-info.md) table of the TiDB cluster, Dumpling automatically adjusts the GC safe point to block GC without affecting the original cluster. However, in either of the following scenarios, Dumpling cannot automatically adjust the GC time: @@ -377,7 +376,7 @@ SET GLOBAL tidb_gc_life_time = '10m'; | `--case-sensitive` | whether table-filter is case-sensitive | false (case-insensitive) | | `-h` or `--host` | The IP address of the connected database host | "127.0.0.1" | | `-t` or `--threads` | The number of concurrent backup threads | 4 | -| `-r` or `--rows` | Enable the in-table concurrency to speed up the export. The default value is `0`, which means disabled. A value greater than 0 means it is enabled, and the value is of `INT` type. When the source database is TiDB, a `-r` value greater than 0 indicates that the TiDB region information is used for splitting, and reduces the memory usage. The specific `-r` value does not affect the split algorithm. When the source database is MySQL and the primary key is of the `INT` type, specifying `-r` can also enable the in-table concurrency. | +| `-r` or `--rows` | Enable the in-table concurrency to speed up the export. The default value is `0`, which means disabled. A value greater than 0 means it is enabled, and the value is of `INT` type. When the source database is TiDB, a `-r` value greater than 0 indicates that the TiDB region information is used for splitting, and reduces the memory usage. The specific `-r` value does not affect the split algorithm. When the source database is MySQL and the primary key or the first column of the composite primary key is of the `INT` type, specifying `-r` can also enable the in-table concurrency. | | `-L` or `--logfile` | Log output address. If it is empty, the log will be output to the console | "" | | `--loglevel` | Log level {debug,info,warn,error,dpanic,panic,fatal} | "info" | | `--logfmt` | Log output format {text,json} | "text" | @@ -401,7 +400,7 @@ SET GLOBAL tidb_gc_life_time = '10m'; | `--cert` | The address of the client certificate file for TLS connection | | `--key` | The address of the client private key file for TLS connection | | `--csv-delimiter` | Delimiter of character type variables in CSV files | '"' | -| `--csv-separator` | Separator of each value in CSV files. It is not recommended to use the default ','. It is recommended to use '\|+\|' or other uncommon character combinations| ',' | ',' | +| `--csv-separator` | Separator for each value in CSV files. If your data contains commas, it is recommended to use a combination of uncommon characters as the separator. Invisible characters are also supported, for example: `--csv-separator $'\001'`. | ',' | | `--csv-null-value` | Representation of null values in CSV files | "\\N" | | `--csv-line-terminator` | The terminator at the end of a line for CSV files. When exporting data to a CSV file, you can specify the desired terminator with this option. This option supports "\\r\\n" and "\\n". The default value is "\\r\\n", which is consistent with the earlier versions. Because quotes in bash have different escaping rules, if you want to specify LF (linefeed) as a terminator, you can use a syntax similar to `--csv-line-terminator $'\n'`. | "\\r\\n" | | `--escape-backslash` | Use backslash (`\`) to escape special characters in the export file | true | diff --git a/dynamic-config.md b/dynamic-config.md index 424225a568d9d..114076bf0caf6 100644 --- a/dynamic-config.md +++ b/dynamic-config.md @@ -1,7 +1,6 @@ --- title: Modify Configuration Dynamically summary: Learn how to dynamically modify the cluster configuration. -aliases: ['/docs/dev/dynamic-config/'] --- # Modify Configuration Dynamically @@ -12,7 +11,7 @@ You can dynamically update the configuration of components (including TiDB, TiKV > **Note:** > -> This feature is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). For TiDB Cloud, you need to contact [TiDB Cloud Support](https://docs.pingcap.com/tidbcloud/tidb-cloud-support) to modify the configurations. +> This feature is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). For TiDB Cloud, you need to contact [TiDB Cloud Support](https://docs.pingcap.com/tidbcloud/tidb-cloud-support) to modify the configurations. ## Common Operations @@ -137,10 +136,6 @@ The following TiKV configuration items can be modified dynamically: | `raftstore.raft-entry-cache-life-time` | The maximum remaining time allowed for the log cache in memory | | `raftstore.split-region-check-tick-interval` | The time interval at which to check whether the Region split is needed | | `raftstore.region-split-check-diff` | The maximum value by which the Region data is allowed to exceed before Region split | -| `raftstore.region-compact-check-interval` | The time interval at which to check whether it is necessary to manually trigger RocksDB compaction | -| `raftstore.region-compact-check-step` | The number of Regions checked at one time for each round of manual compaction | -| `raftstore.region-compact-min-tombstones` | The number of tombstones required to trigger RocksDB compaction | -| `raftstore.region-compact-tombstones-percent` | The proportion of tombstone required to trigger RocksDB compaction | | `raftstore.pd-heartbeat-tick-interval` | The time interval at which a Region's heartbeat to PD is triggered | | `raftstore.pd-store-heartbeat-tick-interval` | The time interval at which a store's heartbeat to PD is triggered | | `raftstore.snap-mgr-gc-tick-interval` | The time interval at which the recycle of expired snapshot files is triggered | @@ -180,8 +175,8 @@ The following TiKV configuration items can be modified dynamically: | `quota.foreground-write-bandwidth` | The soft limit on the bandwidth with which foreground transactions write data | | `quota.foreground-read-bandwidth` | The soft limit on the bandwidth with which foreground transactions and the Coprocessor read data | | `quota.background-cpu-time` | The soft limit on the CPU resources used by TiKV background to process read and write requests | -| `quota.background-write-bandwidth` | The soft limit on the bandwidth with which background transactions write data (not effective yet) | -| `quota.background-read-bandwidth` | The soft limit on the bandwidth with which background transactions and the Coprocessor read data (not effective yet) | +| `quota.background-write-bandwidth` | The soft limit on the bandwidth with which background transactions write data | +| `quota.background-read-bandwidth` | The soft limit on the bandwidth with which background transactions and the Coprocessor read data | | `quota.enable-auto-tune` | Whether to enable the auto-tuning of quota. If this configuration item is enabled, TiKV dynamically adjusts the quota for the background requests based on the load of TiKV instances. | | `quota.max-delay-duration` | The maximum time that a single read or write request is forced to wait before it is processed in the foreground | | `gc.ratio-threshold` | The threshold at which Region GC is skipped (the number of GC versions/the number of keys) | @@ -189,6 +184,12 @@ The following TiKV configuration items can be modified dynamically: | `gc.max-write-bytes-per-sec` | The maximum bytes that can be written into RocksDB per second | | `gc.enable-compaction-filter` | Whether to enable compaction filter | | `gc.compaction-filter-skip-version-check` | Whether to skip the cluster version check of compaction filter (not released) | +| `gc.auto-compaction.check-interval` | The interval at which TiKV checks whether to trigger automatic (RocksDB) compaction | +| `gc.auto-compaction.tombstone-num-threshold` | The number of RocksDB tombstones required to trigger TiKV automatic (RocksDB) compaction | +| `gc.auto-compaction.tombstone-percent-threshold` | The percentage of RocksDB tombstones required to trigger TiKV automatic (RocksDB) compaction | +| `gc.auto-compaction.redundant-rows-threshold` | The number of redundant MVCC rows required to trigger TiKV automatic (RocksDB) compaction | +| `gc.auto-compaction.redundant-rows-percent-threshold` | The percentage of redundant MVCC rows required to trigger TiKV automatic (RocksDB) compaction | +| `gc.auto-compaction.bottommost-level-force` | Whether to force compaction on the bottommost level files in RocksDB | | `{db-name}.max-total-wal-size` | The maximum size of total WAL | | `{db-name}.max-background-jobs` | The number of background threads in RocksDB | | `{db-name}.max-background-flushes` | The maximum number of flush threads in RocksDB | @@ -219,6 +220,11 @@ The following TiKV configuration items can be modified dynamically: | `server.raft-msg-max-batch-size` | Sets the maximum number of Raft messages that are contained in a single gRPC message | | `server.simplify-metrics` | Controls whether to simplify the sampling monitoring metrics | | `storage.block-cache.capacity` | The size of shared block cache (supported since v4.0.3) | +| storage.flow-control.enable | Determines whether to enable the flow control mechanism (dynamic modification is supported starting from v7.5.7) | +| storage.flow-control.memtables-threshold | The maximum number of kvDB memtables that trigger flow control (dynamic modification is supported starting from v7.5.7) | +| storage.flow-control.l0-files-threshold | The maximum number of kvDB L0 files that trigger flow control (dynamic modification is supported starting from v7.5.7) | +| storage.flow-control.soft-pending-compaction-bytes-limit | The threshold of kvDB pending compaction bytes that triggers flow control mechanism to reject some write requests (dynamic modification is supported starting from v7.5.7) | +| storage.flow-control.hard-pending-compaction-bytes-limit | The threshold of kvDB pending compaction bytes that triggers flow control mechanism to reject all write requests (dynamic modification is supported starting from v7.5.7) | | `storage.scheduler-worker-pool-size` | The number of threads in the Scheduler thread pool | | `backup.num-threads` | The number of backup threads (supported since v4.0.3) | | `split.qps-threshold` | The threshold to execute `load-base-split` on a Region. If the QPS of read requests for a Region exceeds `qps-threshold` for 10 consecutive seconds, this Region should be split.| @@ -267,11 +273,12 @@ The following PD configuration items can be modified dynamically: | `cluster-version` | The cluster version | | `schedule.max-merge-region-size` | Controls the size limit of `Region Merge` (in MiB) | | `schedule.max-merge-region-keys` | Specifies the maximum numbers of the `Region Merge` keys | -| `schedule.patrol-region-interval` | Determines the frequency at which `replicaChecker` checks the health state of a Region | +| `schedule.patrol-region-interval` | Determines the frequency at which the checker inspects the health state of a Region | | `schedule.split-merge-interval` | Determines the time interval of performing split and merge operations on the same Region | | `schedule.max-snapshot-count` | Determines the maximum number of snapshots that a single store can send or receive at the same time | | `schedule.max-pending-peer-count` | Determines the maximum number of pending peers in a single store | | `schedule.max-store-down-time` | The downtime after which PD judges that the disconnected store cannot be recovered | +| `schedule.max-store-preparing-time` | Controls the maximum waiting time for the store to go online | | `schedule.leader-schedule-policy` | Determines the policy of Leader scheduling | | `schedule.leader-schedule-limit` | The number of Leader scheduling tasks performed at the same time | | `schedule.region-schedule-limit` | The number of Region scheduling tasks performed at the same time | @@ -289,16 +296,39 @@ The following PD configuration items can be modified dynamically: | `schedule.enable-location-replacement` | Determines whether to enable isolation level check | | `schedule.enable-cross-table-merge` | Determines whether to enable cross-table merge | | `schedule.enable-one-way-merge` | Enables one-way merge, which only allows merging with the next adjacent Region | +| `schedule.region-score-formula-version` | Controls the version of the Region score formula | +| `schedule.scheduler-max-waiting-operator` | Controls the number of waiting operators in each scheduler | +| `schedule.enable-debug-metrics` | Enables the metrics for debugging | +| `schedule.enable-joint-consensus` | Controls whether to use Joint Consensus for replica scheduling | +| `schedule.hot-regions-write-interval` | The time interval at which PD stores hot Region information | +| `schedule.hot-regions-reserved-days` | Specifies how many days the hot Region information is retained | +| `schedule.max-movable-hot-peer-size` | Controls the maximum Region size that can be scheduled for hot Region scheduling | +| `schedule.store-limit-version` | Controls the version of [store limit](/configure-store-limit.md) | | `replication.max-replicas` | Sets the maximum number of replicas | | `replication.location-labels` | The topology information of a TiKV cluster | | `replication.enable-placement-rules` | Enables Placement Rules | | `replication.strictly-match-label` | Enables the label check | +| `replication.isolation-level` | The minimum topological isolation level of a TiKV cluster | | `pd-server.use-region-storage` | Enables independent Region storage | | `pd-server.max-gap-reset-ts` | Sets the maximum interval of resetting timestamp (BR) | | `pd-server.key-type` | Sets the cluster key type | | `pd-server.metric-storage` | Sets the storage address of the cluster metrics | | `pd-server.dashboard-address` | Sets the dashboard address | +| `pd-server.flow-round-by-digit` | Specifies the number of lowest digits to round for the Region flow information | +| `pd-server.min-resolved-ts-persistence-interval` | Determines the interval at which the minimum resolved timestamp is persistent to the PD | +| `pd-server.server-memory-limit` | The memory limit ratio for a PD instance | +| `pd-server.server-memory-limit-gc-trigger` | The threshold ratio at which PD tries to trigger GC | +| `pd-server.enable-gogc-tuner` | Controls whether to enable the GOGC Tuner | +| `pd-server.gc-tuner-threshold` | The maximum memory threshold ratio for tuning GOGC | | `replication-mode.replication-mode` | Sets the backup mode | +| `replication-mode.dr-auto-sync.label-key` | Distinguishes different AZs and needs to match Placement Rules | +| `replication-mode.dr-auto-sync.primary` | The primary AZ | +| `replication-mode.dr-auto-sync.dr` | The disaster recovery (DR) AZ | +| `replication-mode.dr-auto-sync.primary-replicas` | The number of Voter replicas in the primary AZ | +| `replication-mode.dr-auto-sync.dr-replicas` | The number of Voter replicas in the disaster recovery (DR) AZ | +| `replication-mode.dr-auto-sync.wait-store-timeout` | The waiting time for switching to asynchronous replication mode when network isolation or failure occurs | +| `replication-mode.dr-auto-sync.wait-recover-timeout` | The waiting time for switching back to the `sync-recover` status after the network recovers | +| `replication-mode.dr-auto-sync.pause-region-split` | Controls whether to pause Region split operations in the `async_wait` and `async` statuses | For detailed parameter description, refer to [PD Configuration File](/pd-configuration-file.md). @@ -338,10 +368,16 @@ select @@tidb_slow_log_threshold; The following TiDB configuration items can be modified dynamically: | Configuration item | SQL variable | Description | -| :--- | :--- | -| `instance.tidb_enable_slow_log` | `tidb_enable_slow_log` | Whether to enable slow log | -| `instance.tidb_slow_log_threshold` | `tidb_slow_log_threshold` | The threshold of slow log | -| `instance.tidb_expensive_query_time_threshold` | `tidb_expensive_query_time_threshold` | The threshold of a expensive query | +| --- | --- | --- | +| `instance.tidb_enable_slow_log` | `tidb_enable_slow_log` | Controls whether to enable slow log | +| `instance.tidb_slow_log_threshold` | `tidb_slow_log_threshold` | Specifies the threshold of slow log | +| `instance.tidb_expensive_query_time_threshold` | `tidb_expensive_query_time_threshold` | Specifies the threshold of an expensive query | +| `instance.tidb_enable_collect_execution_info` | `tidb_enable_collect_execution_info` | Controls whether to record the execution information of operators | +| `instance.tidb_record_plan_in_slow_log` | `tidb_record_plan_in_slow_log` | Controls whether to record execution plans in the slow log | +| `instance.tidb_force_priority` | `tidb_force_priority` | Specifies the priority of statements that are submitted from this TiDB instance | +| `instance.max_connections` | `max_connections` | Specifies the maximum number of concurrent connections permitted for this TiDB instance | +| `instance.tidb_enable_ddl` | `tidb_enable_ddl` | Controls whether this TiDB instance can become a DDL owner | +| `pessimistic-txn.constraint-check-in-place-pessimistic` | `tidb_constraint_check_in_place_pessimistic` | Controls whether to defer the unique constraint check of a unique index to the next time when this index requires a lock or to the time when the transaction is committed | ### Modify TiFlash configuration dynamically diff --git a/ecosystem-tool-user-case.md b/ecosystem-tool-user-case.md index 8b3380d5f3f56..ef02e74a20ddc 100644 --- a/ecosystem-tool-user-case.md +++ b/ecosystem-tool-user-case.md @@ -1,7 +1,6 @@ --- title: TiDB Tools Use Cases summary: Learn the common use cases of TiDB tools and how to choose the tools. -aliases: ['/docs/dev/ecosystem-tool-user-case/'] --- # TiDB Tools Use Cases diff --git a/ecosystem-tool-user-guide.md b/ecosystem-tool-user-guide.md index 60b1f17d81512..309ae803f6b2e 100644 --- a/ecosystem-tool-user-guide.md +++ b/ecosystem-tool-user-guide.md @@ -1,7 +1,6 @@ --- title: TiDB Tools Overview summary: Learn the tools and applicable scenarios. -aliases: ['/docs/dev/ecosystem-tool-user-guide/','/docs/dev/reference/tools/user-guide/','/docs/dev/how-to/migrate/from-mysql/','/docs/dev/how-to/migrate/incrementally-from-mysql/','/docs/dev/how-to/migrate/overview/'] --- # TiDB Tools Overview diff --git a/enable-tls-between-clients-and-servers.md b/enable-tls-between-clients-and-servers.md index 3114762b92c2d..f1e55c718c6e8 100644 --- a/enable-tls-between-clients-and-servers.md +++ b/enable-tls-between-clients-and-servers.md @@ -1,16 +1,15 @@ --- title: Enable TLS Between TiDB Clients and Servers -summary: Use the encrypted connection to ensure data security. -aliases: ['/docs/dev/enable-tls-between-clients-and-servers/','/docs/dev/how-to/secure/enable-tls-clients/','/docs/dev/encrypted-connections-with-tls-protocols/'] +summary: Use secure connections to ensure data security. --- # Enable TLS between TiDB Clients and Servers -Non-encrypted connection between TiDB's server and clients is allowed by default, which enables third parties that monitor channel traffic to know the data sent and received between the server and the client, including query content and query results. If a channel is untrustworthy (such as if the client is connected to the TiDB server via a public network), then a non-encrypted connection is prone to information leakage. In this case, for security reasons, it is recommended to require an encrypted connection. +By default, TiDB allows insecure connections between the server and clients. This enables third parties that monitor channel traffic to know and possibly modify the data sent and received between the server and the client, including query content and query results. If a channel is untrustworthy (such as if the client is connected to the TiDB server via a public network), an insecure connection is prone to information leakage. In this case, for security reasons, it is recommended to require a connection that is secured with TLS. -The TiDB server supports the encrypted connection based on the TLS (Transport Layer Security). The protocol is consistent with MySQL encrypted connections and is directly supported by existing MySQL clients such as MySQL Client, MySQL Shell and MySQL drivers. TLS is sometimes referred to as SSL (Secure Sockets Layer). Because the SSL protocol has [known security vulnerabilities](https://en.wikipedia.org/wiki/Transport_Layer_Security), TiDB does not support SSL. TiDB supports the following protocols: TLSv1.0, TLSv1.1, TLSv1.2 and TLSv1.3. +The TiDB server supports secure connections based on the TLS (Transport Layer Security) protocol. The protocol is consistent with MySQL secure connections and is directly supported by existing MySQL clients such as MySQL Client, MySQL Shell and MySQL drivers. TLS is sometimes referred to as SSL (Secure Sockets Layer). Because the SSL protocol has [known security vulnerabilities](https://en.wikipedia.org/wiki/Transport_Layer_Security), TiDB does not support SSL. TiDB supports the following protocols: TLSv1.0, TLSv1.1, TLSv1.2 and TLSv1.3. -When an encrypted connection is used, the connection has the following security properties: +When a TLS secured connection is used, the connection has the following security properties: - Confidentiality: the traffic plaintext is encrypted to avoid eavesdropping - Integrity: the traffic plaintext cannot be tampered @@ -20,8 +19,8 @@ To use connections secured with TLS, you first need to configure the TiDB server Similar to MySQL, TiDB allows TLS and non-TLS connections on the same TCP port. For a TiDB server with TLS enabled, you can choose to securely connect to the TiDB server through an encrypted connection, or to use an unencrypted connection. You can use the following ways to require the use of secure connections: -+ Configure the system variable `require_secure_transport` to require secure connections to the TiDB server for all users. -+ Specify `REQUIRE SSL` when you create a user (`create user`), or modify an existing user (`alter user`), which is to specify that specified users must use the encrypted connection to access TiDB. The following is an example of creating a user: ++ Configure the system variable [`require_secure_transport`](/system-variables.md#require_secure_transport-new-in-v610) to require secure connections to the TiDB server for all users. ++ Specify `REQUIRE SSL` when you create a user (`create user`), or modify an existing user (`alter user`), which is to specify that specified users must use TLS connections to access TiDB. The following is an example of creating a user: {{< copyable "sql" >}} @@ -47,27 +46,27 @@ See the following descriptions about the related parameters to enable secure con To enable secure connections with your own certificates in the TiDB server, you must specify both of the `ssl-cert` and `ssl-key` parameters in the configuration file when you start the TiDB server. You can also specify the `ssl-ca` parameter for client authentication (see [Enable authentication](#enable-authentication)). -All the files specified by the parameters are in PEM (Privacy Enhanced Mail) format. Currently, TiDB does not support the import of a password-protected private key, so it is required to provide a private key file without a password. If the certificate or private key is invalid, the TiDB server starts as usual, but the client cannot connect to the TiDB server through an encrypted connection. +All the files specified by the parameters are in PEM (Privacy Enhanced Mail) format. Currently, TiDB does not support the import of a password-protected private key, so it is required to provide a private key file without a password. If the certificate or private key is invalid, the TiDB server starts as usual, but the client cannot connect to the TiDB server through a TLS connection. -If the certificate parameters are correct, TiDB outputs `secure connection is enabled` when started; otherwise, it outputs `secure connection is NOT ENABLED`. +If the certificate parameters are correct, TiDB outputs `mysql protocol server secure connection is enabled` to the logs on `"INFO"` level when started. -For TiDB versions earlier than v5.2.0, you can use `mysql_ssl_rsa_setup --datadir=./certs` to generate certficates. The `mysql_ssl_rsa_setup` tool is a part of MySQL Server. +## Configure the MySQL client to use TLS connections -## Configure the MySQL client to use encrypted connections - -The client of MySQL 5.7 or later versions attempts to establish an encrypted connection by default. If the server does not support encrypted connections, it automatically returns to unencrypted connections. The client of MySQL earlier than version 5.7 uses the unencrypted connection by default. +The client of MySQL 5.7 or later versions attempts to establish a TLS connection by default. If the server does not support TLS connections, it automatically returns to unencrypted connections. The client of MySQL earlier than version 5.7 uses the non-TLS connections by default. You can change the connection behavior of the client using the following `--ssl-mode` parameters: -- `--ssl-mode=REQUIRED`: The client requires an encrypted connection. The connection cannot be established if the server side does not support encrypted connections. -- In the absence of the `--ssl-mode` parameter: The client attempts to use an encrypted connection, but the encrypted connection cannot be established if the server side does not support encrypted connections. Then the client uses an unencrypted connection. +- `--ssl-mode=REQUIRED`: The client requires a TLS connection. The connection cannot be established if the server side does not support TLS connections. +- In the absence of the `--ssl-mode` parameter: The client attempts to use a TLS connection, but the encrypted connection cannot be established if the server side does not support encrypted connections. Then the client uses an unencrypted connection. - `--ssl-mode=DISABLED`: The client uses an unencrypted connection. -MySQL 8.0 clients have two SSL modes in addition to this parameter: +MySQL 8.x clients have two SSL modes in addition to this parameter: - `--ssl-mode=VERIFY_CA`: Validates the certificate from the server against the CA that requires `--ssl-ca`. - `--ssl-mode=VERIFY_IDENTITY`: The same as `VERIFY_CA`, but also validating whether the hostname you are connecting to matches the certificate. +For MySQL 5.7 and MariaDB clients and earlier you can use `--ssl-verify-server-cert` to enable validation of the server certificate. + For more information, see [Client-Side Configuration for Encrypted Connections](https://dev.mysql.com/doc/refman/8.0/en/using-encrypted-connections.html#using-encrypted-connections-client-side-configuration) in MySQL. ## Enable authentication @@ -87,17 +86,15 @@ If the `ssl-ca` parameter is not specified in the TiDB server or MySQL client, t - To perform mutual authentication, meet both of the above requirements. -By default, the server-to-client authentication is optional. Even if the client does not present its certificate of identification during the TLS handshake, the TLS connection can be still established. You can also require the client to be authenticated by specifying `require x509` when creating a user (`create user`), granting permissions (`grant`), or modifying an existing user (`alter user`). The following is an example of creating a user: - -{{< copyable "sql" >}} +By default, the server-to-client authentication is optional. Even if the client does not present its certificate of identification during the TLS handshake, the TLS connection can be still established. You can also require the client to be authenticated by specifying `REQUIRE x509` when creating a user (`CREATE USER`), or modifying an existing user (`ALTER USER`). The following is an example of creating a user: ```sql -create user 'u1'@'%' require x509; +CREATE USER 'u1'@'%' REQUIRE X509; ``` > **Note:** > -> If the login user has configured using the [TiDB Certificate-Based Authentication for Login](/certificate-authentication.md#configure-the-user-certificate-information-for-login-verification), the user is implicitly required to enable the encrypted connection to TiDB. +> If the login user has configured using the [TiDB Certificate-Based Authentication for Login](/certificate-authentication.md#configure-the-user-certificate-information-for-login-verification), the user is implicitly required to enable the TLS connection to TiDB. ## Check whether the current connection uses encryption @@ -105,13 +102,22 @@ Use the `SHOW STATUS LIKE "%Ssl%";` statement to get the details of the current See the following example of the result in an encrypted connection. The results change according to different TLS versions or encryption protocols supported by the client. +```sql +SHOW STATUS LIKE "Ssl%"; +``` + ``` -mysql> SHOW STATUS LIKE "%Ssl%"; -...... -| Ssl_verify_mode | 5 | -| Ssl_version | TLSv1.2 | -| Ssl_cipher | ECDHE-RSA-AES128-GCM-SHA256 | -...... ++-----------------------+-------------------------------------------------------> +| Variable_name | Value > ++-----------------------+-------------------------------------------------------> +| Ssl_cipher | TLS_AES_128_GCM_SHA256 > +| Ssl_cipher_list | RC4-SHA:DES-CBC3-SHA:AES128-SHA:AES256-SHA:AES128-SHA2> +| Ssl_server_not_after | Apr 23 07:59:47 2024 UTC > +| Ssl_server_not_before | Jan 24 07:59:47 2024 UTC > +| Ssl_verify_mode | 5 > +| Ssl_version | TLSv1.3 > ++-----------------------+-------------------------------------------------------> +6 rows in set (0.0062 sec) ``` For the official MySQL client, you can also use the `STATUS` or `\s` statement to view the connection status: @@ -119,13 +125,13 @@ For the official MySQL client, you can also use the `STATUS` or `\s` statement t ``` mysql> \s ... -SSL: Cipher in use is ECDHE-RSA-AES128-GCM-SHA256 +SSL: Cipher in use is TLS_AES_128_GCM_SHA256 ... ``` ## Supported TLS versions, key exchange protocols, and encryption algorithms -The TLS versions, key exchange protocols and encryption algorithms supported by TiDB are determined by the official Golang libraries. +The TLS versions, key exchange protocols and encryption algorithms supported by TiDB are determined by the official Go libraries. The crypto policy for your operating system and the client library you are using might also impact the list of supported protocols and cipher suites. @@ -136,7 +142,7 @@ The crypto policy for your operating system and the client library you are using - TLSv1.2 - TLSv1.3 -The `tls-version` configuration option can be used to limit the TLS versions that can be used. +You can use the [`tls-version`](/tidb-configuration-file.md#tls-version) configuration option to limit the TLS versions that can be used. The actual TLS versions that can be used depend on the OS crypto policy, MySQL client version and the SSL/TLS library that is used by the client. diff --git a/enable-tls-between-components.md b/enable-tls-between-components.md index af73a7887ddfc..bb772f108a835 100644 --- a/enable-tls-between-components.md +++ b/enable-tls-between-components.md @@ -1,7 +1,6 @@ --- title: Enable TLS Between TiDB Components summary: Learn how to enable TLS authentication between TiDB components. -aliases: ['/docs/dev/enable-tls-between-components/','/docs/dev/how-to/secure/enable-tls-between-components/'] --- # Enable TLS Between TiDB Components @@ -84,7 +83,7 @@ Currently, it is not supported to only enable encrypted transmission of some spe - TiFlash (New in v4.0.5) - Configure in the `tiflash.toml` file, and change the `http_port` item to `https_port`: + Configure in the `tiflash.toml` file: ```toml [security] @@ -158,16 +157,17 @@ The Common Name is used for caller verification. In general, the callee needs to To verify component caller's identity, you need to mark the certificate user identity using `Common Name` when generating the certificate, and to check the caller's identity by configuring the `Common Name` list for the callee. +> **Note:** +> +> Currently the `cert-allowed-cn` configuration item of the PD can only be set to one value. Therefore, the `commonName` of all authentication objects must be set to the same value. + - TiDB Configure in the configuration file or command-line arguments: ```toml [security] - cluster-verify-cn = [ - "TiDB-Server", - "TiKV-Control", - ] + cluster-verify-cn = ["TiDB"] ``` - TiKV @@ -176,9 +176,7 @@ To verify component caller's identity, you need to mark the certificate user ide ```toml [security] - cert-allowed-cn = [ - "TiDB-Server", "PD-Server", "TiKV-Control", "RawKvClient1", - ] + cert-allowed-cn = ["TiDB"] ``` - PD @@ -187,7 +185,7 @@ To verify component caller's identity, you need to mark the certificate user ide ```toml [security] - cert-allowed-cn = ["TiKV-Server", "TiDB-Server", "PD-Control"] + cert-allowed-cn = ["TiDB"] ``` - TiFlash (New in v4.0.5) @@ -196,14 +194,14 @@ To verify component caller's identity, you need to mark the certificate user ide ```toml [security] - cert_allowed_cn = ["TiKV-Server", "TiDB-Server"] + cert_allowed_cn = ["TiDB"] ``` Configure in the `tiflash-learner.toml` file: ```toml [security] - cert-allowed-cn = ["PD-Server", "TiKV-Server", "TiFlash-Server"] + cert-allowed-cn = ["TiDB"] ``` ## Reload certificates diff --git a/encryption-at-rest.md b/encryption-at-rest.md index c025f9eec2ba9..e02b9b38b3a18 100644 --- a/encryption-at-rest.md +++ b/encryption-at-rest.md @@ -1,7 +1,6 @@ --- title: Encryption at Rest summary: Learn how to enable encryption at rest to protect sensitive data. -aliases: ['/docs/dev/encryption at rest/'] --- # Encryption at Rest @@ -119,7 +118,7 @@ type = "file" path = "/path/to/key/file" ``` -Here `path` is the path to the key file. The file must contain a 256 bits (or 16 bytes) key encoded as hex string, end with a newline (`\n`) and contain nothing else. Example of the file content: +Here `path` is the path to the key file. The file must contain a 256 bits (or 32 bytes) key encoded as hex string, end with a newline (`\n`) and contain nothing else. Example of the file content: ``` 3b5896b5be691006e0f71c3040a29495ddcad20b14aff61806940ebd780d3c62 @@ -181,7 +180,7 @@ The encryption algorithm currently supported by TiFlash is consistent with that The same master key can be shared by multiple instances of TiFlash, and can also be shared among TiFlash and TiKV. The recommended way to provide a master key in production is via AWS KMS. Alternatively, if using custom key is desired, supplying the master key via file is also supported. The specific method to generate master key and the format of the master key are the same as TiKV. -TiFlash uses the current data key to encrypt all data placed on the disk, including data files, Schmea files, and temporary data files generated during calculations. Data keys are automatically rotated by TiFlash every week by default, and the period is configurable. On key rotation, TiFlash does not rewrite all existing files to replace the key, but background compaction tasks are expected to rewrite old data into new data files, with the most recent data key, if the cluster gets constant write workload. TiFlash keeps track of the key and encryption method used to encrypt each of the files and use the information to decrypt the content on reads. +TiFlash uses the current data key to encrypt all data placed on the disk, including data files, Schema files, and temporary data files generated during calculations. Data keys are automatically rotated by TiFlash every week by default, and the period is configurable. On key rotation, TiFlash does not rewrite all existing files to replace the key, but background compaction tasks are expected to rewrite old data into new data files, with the most recent data key, if the cluster gets constant write workload. TiFlash keeps track of the key and encryption method used to encrypt each of the files and use the information to decrypt the content on reads. ### Key creation diff --git a/error-codes.md b/error-codes.md index 735156891f207..f0b17fb785a5a 100644 --- a/error-codes.md +++ b/error-codes.md @@ -1,7 +1,6 @@ --- title: Error Codes and Troubleshooting summary: Learn about the error codes and solutions in TiDB. -aliases: ['/docs/dev/error-codes/','/docs/dev/reference/error-codes/'] --- # Error Codes and Troubleshooting @@ -10,7 +9,7 @@ This document describes the problems encountered during the use of TiDB and prov ## Error codes -TiDB is compatible with the error codes in MySQL, and in most cases returns the same error code as MySQL. For a list of error codes for MySQL, see [MySQL 5.7 Error Message Reference](https://dev.mysql.com/doc/mysql-errors/5.7/en/). In addition, TiDB has the following unique error codes: +TiDB is compatible with the error codes in MySQL, and in most cases returns the same error code as MySQL. For a list of error codes for MySQL, see [MySQL 8.0 Error Message Reference](https://dev.mysql.com/doc/mysql-errors/8.0/en/). In addition, TiDB has the following unique error codes: > **Note:** > @@ -90,7 +89,7 @@ TiDB is compatible with the error codes in MySQL, and in most cases returns the The single Key-Value pair being written is too large. The largest single Key-Value pair supported in TiDB is 6 MB by default. - If a pair exceeds this limit, you need to properly adjust the [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50) configuration value to relax the limit. + If a pair exceeds this limit, you need to properly adjust the [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v4010-and-v500) configuration value to relax the limit. * Error Number: 8026 @@ -464,7 +463,7 @@ TiDB is compatible with the error codes in MySQL, and in most cases returns the * Error Number: 8228 - Unsupported types are specified when using `setval` on Sequence. + Unsupported types are specified when using `SETVAL` on Sequence. See [Sequence documentation](/sql-statements/sql-statement-create-sequence.md#examples) to find the example of the function. @@ -528,7 +527,7 @@ TiDB is compatible with the error codes in MySQL, and in most cases returns the * Error Number: 9001 - The complete error message: `ERROR 9001 (HY000): PD Server Timeout` + The complete error message: `ERROR 9001 (HY000): PD server timeout` The PD request timed out. @@ -536,7 +535,7 @@ TiDB is compatible with the error codes in MySQL, and in most cases returns the * Error Number: 9002 - The complete error message: `ERROR 9002 (HY000): TiKV Server Timeout` + The complete error message: `ERROR 9002 (HY000): TiKV server timeout` The TiKV request timed out. diff --git a/explain-index-merge.md b/explain-index-merge.md index 0e68f80ffff58..df3a4224eab47 100644 --- a/explain-index-merge.md +++ b/explain-index-merge.md @@ -94,6 +94,6 @@ When using the intersection-type index merge to access tables, the optimizer can > > - If the optimizer can choose the single index scan method (other than full table scan) for a query plan, the optimizer will not automatically use index merge. For the optimizer to use index merge, you need to use the optimizer hint. > -> - Index Merge is not supported in [tempoaray tables](/temporary-tables.md) for now. +> - Index Merge is not supported in [temporary tables](/temporary-tables.md) for now. > > - The intersection-type index merge will not automatically be selected by the optimizer. You must specify the **table name and index name** using the [`USE_INDEX_MERGE`](/optimizer-hints.md#use_index_merget1_name-idx1_name--idx2_name-) hint for it to be selected. diff --git a/explain-joins.md b/explain-joins.md index ceef0558d5bd8..37b5b6e102b04 100644 --- a/explain-joins.md +++ b/explain-joins.md @@ -70,7 +70,7 @@ Index join is efficient in memory usage, but might be slower to execute than oth SELECT * FROM t1 INNER JOIN t2 ON t1.id=t2.t1_id WHERE t1.pad1 = 'value' and t2.pad1='value'; ``` -In an inner join operation, TiDB implements join reordering and might access either `t1` or `t2` first. Assume that TiDB selects `t1` as the first table to apply the `build` step, and then TiDB is able to filter on the predicate `t1.col = 'value'` before probing the table `t2`. The filter for the predicate `t2.col='value'` will be applied on each probe of table `t2`, which might be less efficient than other join methods. +In an inner join operation, TiDB implements join reordering and might access either `t1` or `t2` first. Assume that TiDB selects `t1` as the first table to apply the `build` step, and then TiDB is able to filter on the predicate `t1.pad1 = 'value'` before probing the table `t2`. The filter for the predicate `t2.pad1='value'` will be applied on each probe of table `t2`, which might be less efficient than other join methods. Index join is effective if the build side is small and the probe side is pre-indexed and large. Consider the following query where an index join performs worse than a hash join and is not chosen by the SQL Optimizer: diff --git a/explain-overview.md b/explain-overview.md index 03041376ac3dc..0f422d915e2eb 100644 --- a/explain-overview.md +++ b/explain-overview.md @@ -1,7 +1,6 @@ --- title: TiDB Query Execution Plan Overview summary: Learn about the execution plan information returned by the `EXPLAIN` statement in TiDB. -aliases: ['/docs/dev/query-execution-plan/','/docs/dev/reference/performance/understanding-the-query-execution-plan/','/docs/dev/index-merge/','/docs/dev/reference/performance/index-merge/','/tidb/dev/index-merge','/tidb/dev/query-execution-plan'] --- # TiDB Query Execution Plan Overview diff --git a/explore-htap.md b/explore-htap.md index 01a49469a7dd2..55eb2e1fdcf78 100644 --- a/explore-htap.md +++ b/explore-htap.md @@ -29,7 +29,7 @@ The following are the typical use cases of HTAP: When using TiDB as a data hub, TiDB can meet specific business needs by seamlessly connecting the data for the application and the data warehouse. -For more information about use cases of TiDB HTAP, see [blogs about HTAP on the PingCAP website](https://en.pingcap.com/blog/?tag=htap). +For more information about use cases of TiDB HTAP, see [blogs about HTAP on the PingCAP website](https://www.pingcap.com/blog/?tag=htap). To enhance the overall performance of TiDB, it is recommended to use HTAP in the following technical scenarios: @@ -114,7 +114,7 @@ If any issue occurs during using TiDB, refer to the following documents: - [TiDB cluster troubleshooting guide](/troubleshoot-tidb-cluster.md) - [Troubleshoot a TiFlash Cluster](/tiflash/troubleshoot-tiflash.md) -You are also welcome to create [GitHub Issues](https://github.com/pingcap/tiflash/issues) or submit your questions on [AskTUG](https://asktug.com/). +You are also welcome to create [GitHub Issues](https://github.com/pingcap/tiflash/issues) or ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs). ## What's next diff --git a/expression-syntax.md b/expression-syntax.md index 09489d4f95975..e3dfac5803614 100644 --- a/expression-syntax.md +++ b/expression-syntax.md @@ -1,7 +1,6 @@ --- title: Expression Syntax summary: Learn about the expression syntax in TiDB. -aliases: ['/docs/dev/expression-syntax/','/docs/dev/reference/sql/language-structure/expression-syntax/'] --- # Expression Syntax @@ -18,7 +17,7 @@ The expressions can be divided into the following types: - ParamMarker (`?`), system variables, user variables and CASE expressions. -The following rules are the expression syntax, which is based on the [parser.y](https://github.com/pingcap/parser/blob/master/parser.y) rules of TiDB parser. For the navigable version of the following syntax diagram, refer to [TiDB SQL Syntax Diagram](https://pingcap.github.io/sqlgram/#Expression). +The following rules are the expression syntax, which is based on the [`parser.y`](https://github.com/pingcap/tidb/blob/release-7.5/pkg/parser/parser.y) rules of TiDB parser. For the navigable version of the following syntax diagram, refer to [TiDB SQL Syntax Diagram](https://pingcap.github.io/sqlgram/#Expression). ```ebnf+diagram Expression ::= diff --git a/external-storage-uri.md b/external-storage-uri.md index b9ddf28d1c007..f27429918fa5c 100644 --- a/external-storage-uri.md +++ b/external-storage-uri.md @@ -26,10 +26,10 @@ The basic format of the URI is as follows: - `endpoint`: Specifies the URL of custom endpoint for S3-compatible services (for example, ``). - `force-path-style`: Use path style access rather than virtual hosted style access (defaults to `true`). - `storage-class`: Specifies the storage class of the uploaded objects (for example, `STANDARD` or `STANDARD_IA`). - - `sse`: Specifies the server-side encryption algorithm used to encrypt the uploaded objects (value options: ``, `AES256`, or `aws:kms`). + - `sse`: Specifies the server-side encryption algorithm used to encrypt the uploaded objects (value options: empty, `AES256`, or `aws:kms`). - `sse-kms-key-id`: Specifies the KMS ID if `sse` is set to `aws:kms`. - `acl`: Specifies the canned ACL of the uploaded objects (for example, `private` or `authenticated-read`). - - `role-arn`: When you need to access Amazon S3 data from a third party using a specified [IAM role](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html), you can specify the corresponding [Amazon Resource Name (ARN)](https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) of the IAM role with the `role-arn` URL query parameter, such as `arn:aws:iam::888888888888:role/my-role`. For more information about using an IAM role to access Amazon S3 data from a third party, see [AWS documentation](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_common-scenarios_third-party.html). + - `role-arn`: When you need to access Amazon S3 data from a third party using a specified [IAM role](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html), you can specify the corresponding [Amazon Resource Name (ARN)](https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) of the IAM role with the `role-arn` URL query parameter, such as `arn:aws:iam::888888888888:role/my-role`. For more information about using an IAM role to access Amazon S3 data from a third party, see [AWS documentation](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_common-scenarios_third-party.html). BR does not support this parameter yet. - `external-id`: When you access Amazon S3 data from a third party, you might need to specify a correct [external ID](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user_externalid.html) to assume [the IAM role](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html). In this case, you can use this `external-id` URL query parameter to specify the external ID and make sure that you can assume the IAM role. An external ID is an arbitrary string provided by the third party together with the IAM role ARN to access the Amazon S3 data. Providing an external ID is optional when assuming an IAM role, which means if the third party does not require an external ID for the IAM role, you can assume the IAM role and access the corresponding Amazon S3 data without providing this parameter. The following is an example of an Amazon S3 URI for TiDB Lightning and BR. In this example, you need to specify a specific file path `testfolder`. @@ -38,10 +38,20 @@ The following is an example of an Amazon S3 URI for TiDB Lightning and BR. In th s3://external/testfolder?access-key=${access-key}&secret-access-key=${secret-access-key} ``` +The following is an example of an Amazon S3 URI for TiCDC `sink-uri`. + +```shell +tiup cdc:v{{{ .tidb-version }}} cli changefeed create \ + --server=http://172.16.201.18:8300 \ + --sink-uri="s3://cdc?endpoint=http://10.240.0.38:9000&access-key=${access-key}&secret-access-key=${secret-access-key}" \ + --changefeed-id="cdcTest" \ + --config=cdc_csv.toml +``` + The following is an example of an Amazon S3 URI for [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md). In this example, you need to specify a specific filename `test.csv`. ```shell -s3://external/test.csv?access-key=${access-key}&secret-access-key=${secret-access-key}" +s3://external/test.csv?access-key=${access-key}&secret-access-key=${secret-access-key} ``` ## GCS URI format @@ -79,14 +89,8 @@ gcs://external/test.csv?credentials-file=${credentials-file-path} - `encryption-scope`: Specifies the [encryption scope](https://learn.microsoft.com/en-us/azure/storage/blobs/encryption-scope-manage?tabs=powershell#upload-a-blob-with-an-encryption-scope) for server-side encryption. - `encryption-key`: Specifies the [encryption key](https://learn.microsoft.com/en-us/azure/storage/blobs/encryption-customer-provided-keys) for server-side encryption, which uses the AES256 encryption algorithm. -The following is an example of an Azure Blob Storage URI for TiDB Lightning and BR. In this example, you need to specify a specific file path `testfolder`. +The following is an example of an Azure Blob Storage URI for BR. In this example, you need to specify a specific file path `testfolder`. ```shell azure://external/testfolder?account-name=${account-name}&account-key=${account-key} ``` - -The following is an example of an Azure Blob Storage URI for [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md). In this example, you need to specify a specific filename `test.csv`. - -```shell -azure://external/test.csv?account-name=${account-name}&account-key=${account-key} -``` \ No newline at end of file diff --git a/faq/backup-and-restore-faq.md b/faq/backup-and-restore-faq.md index fb3b489955947..ad837429015fe 100644 --- a/faq/backup-and-restore-faq.md +++ b/faq/backup-and-restore-faq.md @@ -1,7 +1,6 @@ --- title: Backup & Restore FAQs summary: Learn about Frequently Asked Questions (FAQs) and the solutions of backup and restore. -aliases: ['/docs/dev/br/backup-and-restore-faq/','/tidb/dev/pitr-troubleshoot/','/tidb/dev/pitr-known-issues/'] --- # Backup & Restore FAQs @@ -10,7 +9,7 @@ This document lists the frequently asked questions (FAQs) and the solutions of T ## What should I do to quickly recover data after mistakenly deleting or updating data? -TiDB v6.4.0 introduces the flashback feature. You can use this feature to quickly recover data within the GC time to a specified point in time. Therefore, if misoperations occur, you can use this feature to recover data. For details, see [Flashback Cluster](/sql-statements/sql-statement-flashback-to-timestamp.md) and [Flashback Database](/sql-statements/sql-statement-flashback-database.md). +TiDB v6.4.0 introduces the flashback feature. You can use this feature to quickly recover data within the GC time to a specified point in time. Therefore, if misoperations occur, you can use this feature to recover data. For details, see [Flashback Cluster](/sql-statements/sql-statement-flashback-cluster.md) and [Flashback Database](/sql-statements/sql-statement-flashback-database.md). ## In TiDB v5.4.0 and later versions, when backup tasks are performed on the cluster under a heavy workload, why does the speed of backup tasks become slow? @@ -29,7 +28,7 @@ When you perform backup tasks on an offline cluster, to speed up the backup, you ## PITR issues -### What is the difference between [PITR](/br/br-pitr-guide.md) and [cluster flashback](/sql-statements/sql-statement-flashback-to-timestamp.md)? +### What is the difference between [PITR](/br/br-pitr-guide.md) and [cluster flashback](/sql-statements/sql-statement-flashback-cluster.md)? From the perspective of use cases, PITR is usually used to restore the data of a cluster to a specified point in time when the cluster is completely out of service or the data is corrupted and cannot be recovered using other solutions. To use PITR, you need a new cluster for data recovery. The cluster flashback feature is specifically designed for the data error scenarios caused by user mis-operations or other factors, which allows you to restore the data of a cluster in-place to the latest timestamp before the data errors occur. @@ -41,14 +40,6 @@ Currently, the log backup feature is not fully adapted to TiDB Lightning. Theref In upstream clusters where you create log backup tasks, avoid using the TiDB Lightning physical mode to import data. Instead, you can use TiDB Lightning logical mode. If you do need to use the physical mode, perform a snapshot backup after the import is complete, so that PITR can be restored to the time point after the snapshot backup. -### Why is the acceleration of adding indexes feature incompatible with PITR? - -Issue: [#38045](https://github.com/pingcap/tidb/issues/38045) - -Currently, index data created through the [index acceleration](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) feature cannot be backed up by PITR. - -Therefore, after PITR recovery is complete, BR will delete the index data created by index acceleration, and then recreate it. If many indexes are created by index acceleration or the index data is large during the log backup, it is recommended to perform a full backup after creating the indexes. - ### The cluster has recovered from the network partition failure, but the checkpoint of the log backup task progress still does not resume. Why? Issue: [#13126](https://github.com/tikv/tikv/issues/13126) @@ -57,16 +48,6 @@ After a network partition failure in the cluster, the backup task cannot continu To resolve this issue, you need to manually execute the `br log resume` command to resume the log backup task. -### What should I do if the error `execute over region id` is returned when I perform PITR? - -Issue: [#37207](https://github.com/pingcap/tidb/issues/37207) - -This issue usually occurs when you enable log backup during a full data import and afterward perform a PITR to restore data at a time point during the data import. - -Specifically, there is a probability that this issue occurs if there are a large number of hotspot writes for a long time (such as 24 hours) and if the OPS of each TiKV node is larger than 50k/s (you can view the metrics in Grafana: **TiKV-Details** -> **Backup Log** -> **Handle Event Rate**). - -It is recommended that you perform a snapshot backup after the data import and perform PITR based on this snapshot backup. - ## After restoring a downstream cluster using the `br restore point` command, data cannot be accessed from TiFlash. What should I do? Currently, PITR does not support writing data directly to TiFlash during the restore phase. Instead, br command-line tool executes the `ALTER TABLE table_name SET TIFLASH REPLICA ***` DDL to replicate the data. Therefore, TiFlash replicas are not available immediately after PITR completes data restore. Instead, you need to wait for a certain period of time for the data to be replicated from TiKV nodes. To check the replication progress, check the `progress` information in the `INFORMATION_SCHEMA.tiflash_replica` table. @@ -136,9 +117,9 @@ To address this problem, delete the current task using `br log stop`, and then c You can use [`filter.rules`](https://github.com/pingcap/tiflow/blob/7c3c2336f98153326912f3cf6ea2fbb7bcc4a20c/cmd/changefeed.toml#L16) to configure the block list for TiCDC and use [`syncer.ignore-table`](/tidb-binlog/tidb-binlog-configuration-file.md#ignore-table) to configure the block list for Drainer. -### Why is `new_collations_enabled_on_first_bootstrap` mismatch reported during restore? +### Why is `new_collation_enabled` mismatch reported during restore? -Since TiDB v6.0.0, the default value of [`new_collations_enabled_on_first_bootstrap`](/tidb-configuration-file.md#new_collations_enabled_on_first_bootstrap) has changed from `false` to `true`. BR backs up the `new_collations_enabled_on_first_bootstrap` configuration of the upstream cluster and then checks whether the value of this configuration is consistent between the upstream and downstream clusters. If the value is consistent, BR safely restores the data backed up in the upstream cluster to the downstream cluster. If the value is inconsistent, BR does not perform the data restore and reports an error. +Since TiDB v6.0.0, the default value of [`new_collations_enabled_on_first_bootstrap`](/tidb-configuration-file.md#new_collations_enabled_on_first_bootstrap) has changed from `false` to `true`. BR backs up the `new_collation_enabled` configuration in the `mysql.tidb` table of the upstream cluster and then checks whether the value of this configuration is consistent between the upstream and downstream clusters. If the value is consistent, BR safely restores the data backed up in the upstream cluster to the downstream cluster. If the value is inconsistent, BR does not perform the data restore and reports an error. Suppose that you have backed up the data in a TiDB cluster of an earlier version of v6.0.0, and you want to restore this data to a TiDB cluster of v6.0.0 or later versions. In this situation, you need to manually check whether the value of `new_collations_enabled_on_first_bootstrap` is consistent between the upstream and downstream clusters: @@ -169,7 +150,7 @@ To handle this issue, you can try to scale out the cluster resources, reduce the You can try to reduce the number of tables to be created in a batch by setting `--ddl-batch-size` to `128` or a smaller value. -When using BR to restore the backup data with the value of [`--ddl-batch-size`](/br/br-batch-create-table.md#use-batch-create-table) greater than `1`, TiDB writes a DDL job of table creation to the DDL jobs queue that is maintained by TiKV. At this time, the total size of all tables schema sent by TiDB at one time should not exceed 6 MB, because the maximum value of job messages is `6 MB` by default (it is **not recommended** to modify this value. For details, see [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50) and [`raft-entry-max-size`](/tikv-configuration-file.md#raft-entry-max-size)). Therefore, if you set `--ddl-batch-size` to an excessively large value, the schema size of the tables sent by TiDB in a batch at one time exceeds the specified value, which causes BR to report the `entry too large, the max entry size is 6291456, the size of data is 7690800` error. +When using BR to restore the backup data with the value of [`--ddl-batch-size`](/br/br-batch-create-table.md#use-batch-create-table) greater than `1`, TiDB writes a DDL job of table creation to the DDL jobs queue that is maintained by TiKV. At this time, the total size of all tables schema sent by TiDB at one time should not exceed 6 MB, because the maximum value of job messages is `6 MB` by default (it is **not recommended** to modify this value. For details, see [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v4010-and-v500) and [`raft-entry-max-size`](/tikv-configuration-file.md#raft-entry-max-size)). Therefore, if you set `--ddl-batch-size` to an excessively large value, the schema size of the tables sent by TiDB in a batch at one time exceeds the specified value, which causes BR to report the `entry too large, the max entry size is 6291456, the size of data is 7690800` error. ### Where are the backed up files stored when I use `local` storage? @@ -294,7 +275,11 @@ Note that even if you configures [table filter](/table-filter.md#syntax), **BR d - Statistics tables (`mysql.stat_*`). But statistics can be restored. See [Back up statistics](/br/br-snapshot-manual.md#back-up-statistics). - System variable tables (`mysql.tidb`, `mysql.global_variables`) -- [Other system tables](https://github.com/pingcap/tidb/blob/master/br/pkg/restore/systable_restore.go#L31) +- [Other system tables](https://github.com/pingcap/tidb/blob/release-7.5/br/pkg/restore/systable_restore.go#L31) + +### How to deal with the error of `cannot find rewrite rule` during restoration? + +Examine whether there are tables in the restoration cluster sharing the same name as other tables in the backup data but having inconsistent structures. In most cases, this issue is caused by missing indexes in the tables of the restoration cluster. A recommended approach is to delete such tables in the restoration cluster first and then retry restoration. ## Other things you may want to know about backup and restore @@ -327,3 +312,11 @@ If you do not execute `ANALYZE` on the table, TiDB will fail to select the optim ### Does BR back up the `SHARD_ROW_ID_BITS` and `PRE_SPLIT_REGIONS` information of a table? Does the restored table have multiple Regions? Yes. BR backs up the [`SHARD_ROW_ID_BITS` and `PRE_SPLIT_REGIONS`](/sql-statements/sql-statement-split-region.md#pre_split_regions) information of a table. The data of the restored table is also split into multiple Regions. + +## If the recovery process is interrupted, is it necessary to delete the already recovered data and start the recovery again? + +No, it is not necessary. Starting from v7.1.0, BR supports resuming data from a breakpoint. If the recovery is interrupted due to unexpected circumstances, simply restart the recovery task, and it will resume from where it left off. + +## After the recovery is complete, can I delete a specific table and then recover it again? + +Yes, after deleting a specific table, you can recover it again. But note that, you can only recover tables that are deleted using the `DROP TABLE` or `TRUNCATE TABLE` statement, not the `DELETE FROM` statement. This is because `DELETE FROM` only updates the MVCC version to mark the data to be deleted, and the actual data deletion occurs after GC. diff --git a/faq/faq-overview.md b/faq/faq-overview.md index 242079f70da94..a019115075aef 100644 --- a/faq/faq-overview.md +++ b/faq/faq-overview.md @@ -17,27 +17,27 @@ This document summarizes frequently asked questions (FAQs) about TiDB. TiDB architecture and principles - TiDB Architecture FAQs + TiDB Architecture FAQs Deployment - + Data migration @@ -45,31 +45,31 @@ This document summarizes frequently asked questions (FAQs) about TiDB. Data backup and restore - Backup & Restore FAQs + Backup & Restore FAQs SQL operations - SQL FAQs + SQL FAQs Cluster upgrade - TiDB Upgrade FAQs + TiDB Upgrade FAQs Cluster management - Cluster Management FAQs + Cluster Management FAQs Monitor and alert - + High availability and high reliability - + Common error codes - Error Codes and Troubleshooting + Error Codes and Troubleshooting \ No newline at end of file diff --git a/faq/manage-cluster-faq.md b/faq/manage-cluster-faq.md index 7b3a0c618df8a..52428e1cc51a0 100644 --- a/faq/manage-cluster-faq.md +++ b/faq/manage-cluster-faq.md @@ -73,7 +73,7 @@ TiDB provides a few features and [tools](/ecosystem-tool-user-guide.md), with wh The TiDB community is highly active. The engineers have been keeping optimizing features and fixing bugs. Therefore, the TiDB version is updated quite fast. If you want to keep informed of the latest version, see [TiDB Release Timeline](/releases/release-timeline.md). -It is recommeneded to deploy TiDB [using TiUP](/production-deployment-using-tiup.md) or [using TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/stable). TiDB has a unified management of the version number. You can view the version number using one of the following methods: +It is recommended to deploy TiDB [using TiUP](/production-deployment-using-tiup.md) or [using TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/stable). TiDB has a unified management of the version number. You can view the version number using one of the following methods: - `select tidb_version()` - `tidb-server -V` @@ -109,7 +109,7 @@ You can scale TiDB as your business grows. ### If Percolator uses distributed locks and the crash client keeps the lock, will the lock not be released? -For more details, see [Percolator and TiDB Transaction Algorithm](https://pingcap.com/blog-cn/percolator-and-txn/) in Chinese. +For more details, see [Percolator and TiDB Transaction Algorithm](https://pingkai.cn/tidbcommunity/blog/f537be2c) in Chinese. ### Why does TiDB use gRPC instead of Thrift? Is it because Google uses it? @@ -145,6 +145,10 @@ Most of the APIs of PD are available only when the TiKV cluster is initialized. This is because the `--initial-cluster` in the PD startup parameter contains a member that doesn't belong to this cluster. To solve this problem, check the corresponding cluster of each member, remove the wrong member, and then restart PD. +### The `[PD:encryption:ErrEncryptionNewMasterKey]fail to get encryption key from file /root/path/file%!(EXTRA string=open /root/path/file: permission denied)` message is displayed when enabling encryption at rest for PD + +Encryption at rest does not support storing the key file in the `root` directory or its subdirectories. Even if you grant read permissions, the same error occurs. To resolve this issue, store the key file in a location outside the `root` directory. + ### What's the maximum tolerance for time synchronization error of PD? PD can tolerate any synchronization error, but a larger error value means a larger gap between the timestamp allocated by the PD and the physical time, which will affect functions such as read of historical versions. @@ -411,7 +415,10 @@ The memory usage of TiKV mainly comes from the block-cache of RocksDB, which is ### Can both TiDB data and RawKV data be stored in the same TiKV cluster? -No. TiDB (or data created from the transactional API) relies on a specific key format. It is not compatible with data created from RawKV API (or data from other RawKV-based services). +It depends on your TiDB version and whether TiKV API V2 is enabled ([`storage.api-version = 2`](/tikv-configuration-file.md#api-version-new-in-v610)). + +- If your TiDB version is v6.1.0 or later and TiKV API V2 is enabled, TiDB data and RawKV data can be stored in the same TiKV cluster. +- Otherwise, the answer is no because the key format of TiDB data (or data created using the transactional API) is incompatible with data created using the RawKV API (or data from other RawKV-based services). ## TiDB testing diff --git a/faq/migration-tidb-faq.md b/faq/migration-tidb-faq.md index 1143d50d43d9b..abdc1b797ce8d 100644 --- a/faq/migration-tidb-faq.md +++ b/faq/migration-tidb-faq.md @@ -92,7 +92,7 @@ To migrate all the data or migrate incrementally from DB2 or Oracle to TiDB, see Currently, it is recommended to use OGG. -### Error: `java.sql.BatchUpdateExecption:statement count 5001 exceeds the transaction limitation` while using Sqoop to write data into TiDB in `batches` +### Error: `java.sql.BatchUpdateException:statement count 5001 exceeds the transaction limitation` while using Sqoop to write data into TiDB in `batches` In Sqoop, `--batch` means committing 100 `statement`s in each batch, but by default each `statement` contains 100 SQL statements. So, 100 * 100 = 10000 SQL statements, which exceeds 5000, the maximum number of statements allowed in a single TiDB transaction. @@ -146,7 +146,7 @@ The total read capacity has no limit. You can increase the read capacity by addi ### The error message `transaction too large` is displayed -Due to the limitation of the underlying storage engine, each key-value entry (one row) in TiDB should be no more than 6MB. You can adjust the [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50) configuration value up to 120MB. +Due to the limitation of the underlying storage engine, each key-value entry (one row) in TiDB should be no more than 6MB. You can adjust the [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v4010-and-v500) configuration value up to 120MB. Distributed transactions need two-phase commit and the bottom layer performs the Raft replication. If a transaction is very large, the commit process would be quite slow and the write conflict is more likely to occur. Moreover, the rollback of a failed transaction leads to an unnecessary performance penalty. To avoid these problems, we limit the total size of key-value entries to no more than 100MB in a transaction by default. If you need larger transactions, modify the value of `txn-total-size-limit` in the TiDB configuration file. The maximum value of this configuration item is up to 10G. The actual limitation is also affected by the physical memory of the machine. @@ -170,13 +170,13 @@ Yes. ### Why does the query speed getting slow after deleting data? -Deleting a large amount of data leaves a lot of useless keys, affecting the query efficiency. Currently the Region Merge feature is in development, which is expected to solve this problem. For details, see the [deleting data section in TiDB Best Practices](https://en.pingcap.com/blog/tidb-best-practice/#write). +Deleting a large amount of data leaves a lot of useless keys, affecting the query efficiency. Currently the Region Merge feature is in development, which is expected to solve this problem. For details, see the [deleting data section in TiDB Best Practices](https://www.pingcap.com/blog/tidb-best-practice/#write). ### What is the most efficient way of deleting data? When deleting a large amount of data, it is recommended to use `Delete from t where xx limit 5000;`. It deletes through the loop and uses `Affected Rows == 0` as a condition to end the loop, so as not to exceed the limit of transaction size. With the prerequisite of meeting business filtering logic, it is recommended to add a strong filter index column or directly use the primary key to select the range, such as `id >= 5000*n+m and id < 5000*(n+1)+m`. -If the amount of data that needs to be deleted at a time is very large, this loop method will get slower and slower because each deletion traverses backward. After deleting the previous data, lots of deleted flags remain for a short period (then all will be processed by Garbage Collection) and influence the following Delete statement. If possible, it is recommended to refine the Where condition. See [details in TiDB Best Practices](https://en.pingcap.com/blog/tidb-best-practice/#write). +If the amount of data that needs to be deleted at a time is very large, this loop method will get slower and slower because each deletion traverses backward. After deleting the previous data, lots of deleted flags remain for a short period (then all will be processed by Garbage Collection) and influence the following Delete statement. If possible, it is recommended to refine the Where condition. See [details in TiDB Best Practices](https://www.pingcap.com/blog/tidb-best-practice/#write). ### How to improve the data loading speed in TiDB? diff --git a/faq/monitor-faq.md b/faq/monitor-faq.md index 5cfa36a2781cf..8b015916823d8 100644 --- a/faq/monitor-faq.md +++ b/faq/monitor-faq.md @@ -18,7 +18,7 @@ The monitoring system of TiDB consists of Prometheus and Grafana. From the dashb Yes. Find the startup script on the machine where Prometheus is started, edit the startup parameter and restart Prometheus. -```config +``` --storage.tsdb.retention="60d" ``` diff --git a/faq/sql-faq.md b/faq/sql-faq.md index 8b1eeacaf5478..8e098d446ba5b 100644 --- a/faq/sql-faq.md +++ b/faq/sql-faq.md @@ -32,7 +32,9 @@ In addition, you can also use the [SQL binding](/sql-plan-management.md#sql-bind ## How to prevent the execution of a particular SQL statement? -You can create [SQL bindings](/sql-plan-management.md#sql-binding) with the [`MAX_EXECUTION_TIME`](/optimizer-hints.md#max_execution_timen) hint to limit the execution time of a particular statement to a small value (for example, 1ms). In this way, the statement is terminated automatically by the threshold. +For TiDB v7.5.0 or later versions, you can use the [`QUERY WATCH`](/sql-statements/sql-statement-query-watch.md) statement to terminate specific SQL statements. For more details, see [Manage queries that consume more resources than expected (Runaway Queries)](/tidb-resource-control.md#query-watch-parameters). + +For versions earlier than TiDB v7.5.0, you can create [SQL bindings](/sql-plan-management.md#sql-binding) with the [`MAX_EXECUTION_TIME`](/optimizer-hints.md#max_execution_timen) hint to limit the execution time of a particular statement to a small value (for example, 1ms). In this way, the statement is terminated automatically by the threshold. For example, to prevent the execution of `SELECT * FROM t1, t2 WHERE t1.id = t2.id`, you can use the following SQL binding to limit the execution time of the statement to 1ms: @@ -132,7 +134,7 @@ For details, see [description of the `SELECT` syntax elements](/sql-statements/s ## Can the codec of TiDB guarantee that the UTF-8 string is memcomparable? Is there any coding suggestion if our key needs to support UTF-8? -TiDB uses the UTF-8 character set by default and currently only supports UTF-8. The string of TiDB uses the memcomparable format. +The default character set in TiDB is `utf8mb4`. The strings are in memcomparable format. For more information about the character set in TiDB, see [Character Set and Collation](/character-set-and-collation.md). ## What is the maximum number of statements in a transaction? @@ -151,7 +153,7 @@ TiDB supports modifying the [`sql_mode`](/system-variables.md#sql_mode) system v - Changes to [`GLOBAL`](/sql-statements/sql-statement-set-variable.md) scoped variables propagate to the rest servers of the cluster and persist across restarts. This means that you do not need to change the `sql_mode` value on each TiDB server. - Changes to `SESSION` scoped variables only affect the current client session. After restarting a server, the changes are lost. -## Error: `java.sql.BatchUpdateExecption:statement count 5001 exceeds the transaction limitation` while using Sqoop to write data into TiDB in batches +## Error: `java.sql.BatchUpdateException:statement count 5001 exceeds the transaction limitation` while using Sqoop to write data into TiDB in batches In Sqoop, `--batch` means committing 100 statements in each batch, but by default each statement contains 100 SQL statements. So, 100 * 100 = 10000 SQL statements, which exceeds 5000, the maximum number of statements allowed in a single TiDB transaction. @@ -184,7 +186,7 @@ None of the `DELETE`, `TRUNCATE` and `DROP` operations release data immediately. ## Why does the query speed get slow after data is deleted? -Deleting a large amount of data leaves a lot of useless keys, affecting the query efficiency. To solve the problem, you can use the [Region Merge](/best-practices/massive-regions-best-practices.md#method-3-enable-region-merge) feature. For details, see the [deleting data section in TiDB Best Practices](https://en.pingcap.com/blog/tidb-best-practice/#write). +Deleting a large amount of data leaves a lot of useless keys, affecting the query efficiency. To solve the problem, you can use the [Region Merge](/best-practices/massive-regions-best-practices.md#method-3-enable-region-merge) feature. For details, see the [deleting data section in TiDB Best Practices](https://www.pingcap.com/blog/tidb-best-practice/#write). ## What should I do if it is slow to reclaim storage space after deleting data? @@ -207,6 +209,10 @@ TiDB supports changing the priority on a [global](/system-variables.md#tidb_forc - `DELAYED`: this statement has normal priority and is the same as the `NO_PRIORITY` setting for `tidb_force_priority`. +> **Note:** +> +> Starting from v6.6.0, TiDB supports [Resource Control](/tidb-resource-control.md). You can use this feature to execute SQL statements with different priorities in different resource groups. By configuring proper quotas and priorities for these resource groups, you can gain better scheduling control for SQL statements with different priorities. When resource control is enabled, statement priority will no longer take effect. It is recommended that you use [Resource Control](/tidb-resource-control.md) to manage resource usage for different SQL statements. + You can combine the above two parameters with the DML of TiDB to use them. For example: 1. Adjust the priority by writing SQL statements in the database: @@ -225,11 +231,11 @@ You can combine the above two parameters with the DML of TiDB to use them. For e ## What's the trigger strategy for `auto analyze` in TiDB? -Trigger strategy: `auto analyze` is automatically triggered when the number of rows in a new table reaches 1000 and this table has no write operation within one minute. +When the number of rows in a table or a single partition of a partitioned table reaches 1000, and the ratio (the number of modified rows / the current total number of rows) of the table or partition is larger than [`tidb_auto_analyze_ratio`](/system-variables.md#tidb_auto_analyze_ratio), the [`ANALYZE`](/sql-statements/sql-statement-analyze-table.md) statement is automatically triggered. -When the ratio (the number of modified rows / the current total number of rows) is larger than `tidb_auto_analyze_ratio`, the `analyze` statement is automatically triggered. The default value of `tidb_auto_analyze_ratio` is 0.5, indicating that this feature is enabled by default. To ensure safety, its minimum value is 0.3 when the feature is enabled, and it must be smaller than `pseudo-estimate-ratio` whose default value is 0.8, otherwise pseudo statistics will be used for a period of time. It is recommended to set `tidb_auto_analyze_ratio` to 0.5. +The default value of the `tidb_auto_analyze_ratio` system variable is `0.5`, indicating that this feature is enabled by default. It is not recommended to set the value of `tidb_auto_analyze_ratio` to be larger than or equal to [`pseudo-estimate-ratio`](/tidb-configuration-file.md#pseudo-estimate-ratio) (the default value is `0.8`), otherwise the optimizer might use pseudo statistics. TiDB v5.3.0 introduces the [`tidb_enable_pseudo_for_outdated_stats`](/system-variables.md#tidb_enable_pseudo_for_outdated_stats-new-in-v530) variable, and when you set it to `OFF`, pseudo statistics are not used even if the statistics are outdated. -To disable auto analyze, use the system variable `tidb_enable_auto_analyze`. +To disable `auto analyze`, use the system variable [`tidb_enable_auto_analyze`](/system-variables.md#tidb_enable_auto_analyze-new-in-v610). ## Can I use optimizer hints to override the optimizer behavior? @@ -241,7 +247,7 @@ SELECT column_name FROM table_name USE INDEX(index_name)WHERE where_conditio ## DDL Execution -This section lists issues related to DDL statement execution. For detailed explanations on the DDL execution principles, see [Execution Principles and Best Practices of DDL Statements](/ddl-introduction.md). +This section lists issues related to DDL statement execution. For detailed explanations on the DDL execution principles, see [Execution Principles and Best Practices of DDL Statements](/best-practices/ddl-introduction.md). ### How long does it take to perform various DDL operations? @@ -294,7 +300,7 @@ In the preceding causes, only Cause 1 is related to tables. Cause 1 and Cause 2 ### What are the causes of the "Information schema is out of date" error? -Before TiDB v6.5.0, when executing a DML statement, if TiDB fails to load the latest schema within a DDL lease (45s by default), the `Information schema is out of date` error might occur. Possible causes are: +When executing a DML statement, if TiDB fails to load the latest schema within a DDL lease (45s by default), the `Information schema is out of date` error might occur. Possible causes are as follows: - The TiDB instance that executed this DML was killed, and the transaction execution corresponding to this DML statement took longer than a DDL lease. When the transaction was committed, the error occurred. - TiDB failed to connect to PD or TiKV while executing this DML statement. As a result, TiDB failed to load schema within a DDL lease or disconnected from PD due to the keepalive setting. @@ -331,6 +337,73 @@ Whether your cluster is a new cluster or an upgraded cluster from an earlier ver - If the owner does not exist, try manually triggering owner election with: `curl -X POST http://{TiDBIP}:10080/ddl/owner/resign`. - If the owner exists, export the Goroutine stack and check for the possible stuck location. +## Collation used in JDBC connections + +This section lists questions related to collations used in JDBC connections. For information about character sets and collations supported by TiDB, see [Character Set and Collation](/character-set-and-collation.md). + +### What collation is used in a JDBC connection when `connectionCollation` is not configured in the JDBC URL? + +When `connectionCollation` is not configured in the JDBC URL, there are two scenarios: + +**Scenario 1**: Neither `connectionCollation` nor `characterEncoding` is configured in the JDBC URL + +- For Connector/J 8.0.25 and earlier versions, the JDBC driver attempts to use the server's default character set. Because the default character set of TiDB is `utf8mb4`, the driver uses `utf8mb4_bin` as the connection collation. +- For Connector/J 8.0.26 and later versions, the JDBC driver uses the `utf8mb4` character set and automatically selects the collation based on the return value of `SELECT VERSION()`. + + - When the return value is less than `8.0.1`, the driver uses `utf8mb4_general_ci` as the connection collation. TiDB follows the driver and uses `utf8mb4_general_ci` as the collation. + - When the return value is greater than or equal to `8.0.1`, the driver uses `utf8mb4_0900_ai_ci` as the connection collation. TiDB v7.4.0 and later versions follow the driver and use `utf8mb4_0900_ai_ci` as the collation, while TiDB versions earlier than v7.4.0 fall back to using the default collation `utf8mb4_bin` because the `utf8mb4_0900_ai_ci` collation is not supported in these versions. + +**Scenario 2**: `characterEncoding=utf8` is configured in the JDBC URL but `connectionCollation` is not configured. The JDBC driver uses the `utf8mb4` character set according to the mapping rules. The collation is determined according to the rules described in scenario 1. + +### How to handle collation changes after upgrading TiDB? + +In TiDB v7.4 and earlier versions, if `connectionCollation` is not configured, and `characterEncoding` is either not configured or set to `UTF-8` in the JDBC URL, the TiDB [`collation_connection`](/system-variables.md#collation_connection) variable defaults to the `utf8mb4_bin` collation. + +Starting from TiDB v7.4, if `connectionCollation` is not configured, and `characterEncoding` is either not configured or set to `UTF-8` in the JDBC URL, the value of the [`collation_connection`](/system-variables.md#collation_connection) variable depends on the JDBC driver version. For more information, see [Collation used in JDBC connections](#what-collation-is-used-in-a-jdbc-connection-when-connectioncollation-is-not-configured-in-the-jdbc-url). + +When upgrading from an earlier version to v7.4 or later (for example, from v6.5 to v7.5), if you need to maintain the `collation_connection` as `utf8mb4_bin` for JDBC connections, it is recommended to configure the `connectionCollation` parameter in the JDBC URL. + +The following is a common JDBC URL configuration in TiDB v6.5: + +``` +spring.datasource.url=JDBC:mysql://{TiDBIP}:{TiDBPort}/{DBName}?characterEncoding=UTF-8&useSSL=false&useServerPrepStmts=true&cachePrepStmts=true&prepStmtCacheSqlLimit=10000&prepStmtCacheSize=1000&useConfigs=maxPerformance&rewriteBatchedStatements=true&defaultFetchSize=-2147483648&allowMultiQueries=true +``` + +After upgrading to TiDB v7.5 or a later version, it is recommended to configure the `connectionCollation` parameter in the JDBC URL: + +``` +spring.datasource.url=JDBC:mysql://{TiDBIP}:{TiDBPort}/{DBName}?characterEncoding=UTF-8&connectionCollation=utf8mb4_bin&useSSL=false&useServerPrepStmts=true&cachePrepStmts=true&prepStmtCacheSqlLimit=10000&prepStmtCacheSize=1000&useConfigs=maxPerformance&rewriteBatchedStatements=true&defaultFetchSize=-2147483648&allowMultiQueries=true +``` + +### What are the differences between the `utf8mb4_bin` and `utf8mb4_0900_ai_ci` collations? + +| Collation | Case-sensitive | Ignore trailing spaces | Accent-sensitive | Comparison method | +|----------------------|----------------|------------------|--------------|------------------------| +| `utf8mb4_bin` | Yes | Yes | Yes | Compare binary values | +| `utf8mb4_0900_ai_ci` | No | No | No | Use Unicode sorting algorithm | + +For example: + +```sql +-- utf8mb4_bin is case-sensitive +SELECT 'apple' = 'Apple' COLLATE utf8mb4_bin; -- Returns 0 (FALSE) + +-- utf8mb4_0900_ai_ci is case-insensitive +SELECT 'apple' = 'Apple' COLLATE utf8mb4_0900_ai_ci; -- Returns 1 (TRUE) + +-- utf8mb4_bin ignores trailing spaces +SELECT 'Apple ' = 'Apple' COLLATE utf8mb4_bin; -- Returns 1 (TRUE) + +-- utf8mb4_0900_ai_ci does not ignore trailing spaces +SELECT 'Apple ' = 'Apple' COLLATE utf8mb4_0900_ai_ci; -- Returns 0 (FALSE) + +-- utf8mb4_bin is accent-sensitive +SELECT 'café' = 'cafe' COLLATE utf8mb4_bin; -- Returns 0 (FALSE) + +-- utf8mb4_0900_ai_ci is accent-insensitive +SELECT 'café' = 'cafe' COLLATE utf8mb4_0900_ai_ci; -- Returns 1 (TRUE) +``` + ## SQL optimization ### TiDB execution plan description diff --git a/faq/tidb-faq.md b/faq/tidb-faq.md index 4668ff9d06322..98f5ec6a3e594 100644 --- a/faq/tidb-faq.md +++ b/faq/tidb-faq.md @@ -1,7 +1,6 @@ --- title: TiDB Architecture FAQs summary: Learn about the most frequently asked questions (FAQs) relating to TiDB. -aliases: ['/docs/dev/faq/tidb-faq/','/docs/dev/faq/tidb/','/docs/dev/tiflash/tiflash-faq/','/docs/dev/reference/tiflash/faq/','/tidb/dev/tiflash-faq'] --- # TiDB Architecture FAQs @@ -44,7 +43,7 @@ Yes, it is. When all the required services are started, you can use TiDB as easi ### How is TiDB compatible with MySQL? -Currently, TiDB supports the majority of MySQL 5.7 syntax, but does not support triggers, stored procedures, and user-defined functions. For more details, see [Compatibility with MySQL](/mysql-compatibility.md). +Currently, TiDB supports the majority of MySQL 8.0 syntax, but does not support triggers, stored procedures, and user-defined functions. For more details, see [Compatibility with MySQL](/mysql-compatibility.md). ### Does TiDB support distributed transactions? diff --git a/faq/upgrade-faq.md b/faq/upgrade-faq.md index ce6be8527ed2a..d9c6a1589ac36 100644 --- a/faq/upgrade-faq.md +++ b/faq/upgrade-faq.md @@ -1,7 +1,6 @@ --- title: Upgrade and After Upgrade FAQs summary: Learn about some FAQs and the solutions during and after upgrading TiDB. -aliases: ['/docs/dev/faq/upgrade-faq/','/docs/dev/faq/upgrade/'] --- # Upgrade and After Upgrade FAQs @@ -36,6 +35,12 @@ It is not recommended to upgrade TiDB using the binary. Instead, it is recommend This section lists some FAQs and their solutions after you upgrade TiDB. +### The collation in JDBC connections changes after upgrading TiDB + +When upgrading from an earlier version to v7.4 or later, if the `connectionCollation` is not configured, and the `characterEncoding` is either not configured or configured as `UTF-8` in the JDBC URL, the default collation in your JDBC connections might change from `utf8mb4_bin` to `utf8mb4_0900_ai_ci` after upgrading. If you need to maintain the collation as `utf8mb4_bin`, configure `connectionCollation=utf8mb4_bin` in the JDBC URL. + +For more information, see [Collation used in JDBC connections](/faq/sql-faq.md#collation-used-in-jdbc-connections). + ### The character set (charset) errors when executing DDL operations In v2.1.0 and earlier versions (including all versions of v2.0), the character set of TiDB is UTF-8 by default. But starting from v2.1.1, the default character set has been changed into UTF8MB4. diff --git a/filter-binlog-event.md b/filter-binlog-event.md index a2e8b4186515f..21eef40d97821 100644 --- a/filter-binlog-event.md +++ b/filter-binlog-event.md @@ -7,8 +7,8 @@ summary: Learn how to filter binlog events when migrating data. This document describes how to filter binlog events when you use DM to perform continuous incremental data replication. For the detailed replication instructions, refer to the following documents by scenarios: -- [Migrate MySQL of Small Datasets to TiDB](/migrate-small-mysql-to-tidb.md) -- [Migrate MySQL of Large Datasets to TiDB](/migrate-large-mysql-to-tidb.md) +- [Migrate Small Datasets from MySQL to TiDB](/migrate-small-mysql-to-tidb.md) +- [Migrate Large Datasets from MySQL to TiDB](/migrate-large-mysql-to-tidb.md) - [Migrate and Merge MySQL Shards of Small Datasets to TiDB](/migrate-small-mysql-shards-to-tidb.md) - [Migrate and Merge MySQL Shards of Large Datasets to TiDB](/migrate-large-mysql-shards-to-tidb.md) diff --git a/filter-dml-event.md b/filter-dml-event.md index 9d8212c33407c..777708a021ee4 100644 --- a/filter-dml-event.md +++ b/filter-dml-event.md @@ -7,8 +7,8 @@ summary: Learn how to filter DML events using SQL expressions. This document introduces how to filter binlog events using SQL expressions when you use DM to perform continuous incremental data replication. For the detailed replication instruction, refer to the following documents: -- [Migrate MySQL of Small Datasets to TiDB](/migrate-small-mysql-to-tidb.md) -- [Migrate MySQL of Large Datasets to TiDB](/migrate-large-mysql-to-tidb.md) +- [Migrate Small Datasets from MySQL to TiDB](/migrate-small-mysql-to-tidb.md) +- [Migrate Large Datasets from MySQL to TiDB](/migrate-large-mysql-to-tidb.md) - [Migrate and Merge MySQL Shards of Small Datasets to TiDB](/migrate-small-mysql-shards-to-tidb.md) - [Migrate and Merge MySQL Shards of Large Datasets to TiDB](/migrate-large-mysql-shards-to-tidb.md) diff --git a/follower-read.md b/follower-read.md index 2d3e76e2a958e..43d994b39ac2a 100644 --- a/follower-read.md +++ b/follower-read.md @@ -1,7 +1,6 @@ --- title: Follower Read summary: This document describes the use and implementation of Follower Read. -aliases: ['/docs/dev/follower-read/','/docs/dev/reference/performance/follower-read/'] --- # Follower Read @@ -32,23 +31,28 @@ Default: leader This variable is used to set the expected data read mode. -- When the value of `tidb_replica_read` is set to `leader` or an empty string, TiDB maintains its default behavior and sends all read operations to the leader replica to perform. -- When the value of `tidb_replica_read` is set to `follower`, TiDB selects a follower replica of the Region to perform all read operations. -- When the value of `tidb_replica_read` is set to `leader-and-follower`, TiDB can select any replicas to perform read operations. In this mode, read requests are load balanced between the leader and follower. +- When you set the value of `tidb_replica_read` to `leader` or an empty string, TiDB maintains its default behavior and sends all read operations to the leader replica to perform. +- When you set the value of `tidb_replica_read` to `follower`, TiDB selects a follower replica of the Region to perform read operations. If the Region has learner replicas, TiDB also considers them for reads with the same priority. If no available follower or learner replicas exist for the current Region, TiDB reads from the leader replica. +- When the value of `tidb_replica_read` is set to `leader-and-follower`, TiDB can select any replicas to perform read operations. In this mode, read requests are load balanced between the leader, follower, and learner. - When the value of `tidb_replica_read` is set to `prefer-leader`, TiDB prefers to select the leader replica to perform read operations. If the leader replica is obviously slow in processing read operations (such as caused by disk or network performance jitter), TiDB will select other available follower replicas to perform read operations. -- When the value of `tidb_replica_read` is set to `closest-replicas`, TiDB prefers to select a replica in the same availability zone to perform read operations, which can be a leader or a follower. If there is no replica in the same availability zone, TiDB reads from the leader replica. +- When the value of `tidb_replica_read` is set to `closest-replicas`, TiDB prefers to select a replica in the same availability zone to perform read operations, which can be a leader, a follower, or a learner. If there is no replica in the same availability zone, TiDB reads from the leader replica. - When the value of `tidb_replica_read` is set to `closest-adaptive`: - If the estimated result of a read request is greater than or equal to the value of [`tidb_adaptive_closest_read_threshold`](/system-variables.md#tidb_adaptive_closest_read_threshold-new-in-v630), TiDB prefers to select a replica in the same availability zone for read operations. To avoid unbalanced distribution of read traffic across availability zones, TiDB dynamically detects the distribution of availability zones for all online TiDB and TiKV nodes. In each availability zone, the number of TiDB nodes whose `closest-adaptive` configuration takes effect is limited, which is always the same as the number of TiDB nodes in the availability zone with the fewest TiDB nodes, and the other TiDB nodes automatically read from the leader replica. For example, if TiDB nodes are distributed across 3 availability zones (A, B, and C), where A and B each contains 3 TiDB nodes and C contains only 2 TiDB nodes, the number of TiDB nodes whose `closest-adaptive` configuration takes effect in each availability zone is 2, and the other TiDB node in each of the A and B availability zones automatically selects the leader replica for read operations. - If the estimated result of a read request is less than the value of [`tidb_adaptive_closest_read_threshold`](/system-variables.md#tidb_adaptive_closest_read_threshold-new-in-v630), TiDB can only select the leader replica for read operations. -- When the value of `tidb_replica_read` is set to `learner`, TiDB reads data from the learner replica. If there is no learner replica in the Region, TiDB returns an error. +- When you set the value of `tidb_replica_read` to `learner`, TiDB reads data from the learner replica. If no learner replica is available for the current Region, TiDB reads from an available leader or follower replica. > **Note:** > -> When the value of `tidb_replica_read` is set to `closest-replicas` or `closest-adaptive`, you need to configure the cluster to ensure that replicas are distributed across availability zones according to the specified configuration. To configure `location-labels` for PD and set the correct `labels` for TiDB and TiKV, refer to [Schedule replicas by topology labels](/schedule-replicas-by-topology-labels.md). TiDB depends on the `zone` label to match TiKV nodes in the same availability zone, so you need to make sure that the `zone` label is included in the `location-labels` of PD and `zone` is included in the configuration of each TiDB and TiKV node. If your cluster is deployed using TiDB Operator, refer to [High availability of data](https://docs.pingcap.com/tidb-in-kubernetes/v1.4/configure-a-tidb-cluster#high-availability-of-data). +> When you set `tidb_replica_read` to `closest-replicas` or `closest-adaptive`, to ensure that replicas are distributed across availability zones according to the specified configuration, you need to configure `location-labels` for PD and set the correct `labels` for TiDB and TiKV according to [Schedule replicas by topology labels](/schedule-replicas-by-topology-labels.md). TiDB depends on the `zone` label to match TiKV nodes in the same availability zone, so you need to make sure that the `zone` label is included in the `location-labels` of PD and `zone` is included in the configuration of each TiDB and TiKV node. If your cluster is deployed using TiDB Operator, refer to [High availability of data](https://docs.pingcap.com/tidb-in-kubernetes/stable/configure-a-tidb-cluster#high-availability-of-data). +> +> For TiDB v7.5.0 and earlier versions: +> +> - If you set `tidb_replica_read` to `follower` and no follower or learner replicas are available, TiDB returns an error. +> - If you set `tidb_replica_read` to `learner` and no learner replicas are available, TiDB returns an error. @@ -66,4 +70,4 @@ When the follower node processes a read request, it first uses `ReadIndex` of th Because the Follower Read feature does not affect TiDB's Snapshot Isolation transaction isolation level, TiDB adopts the round-robin strategy to select the follower replica. Currently, for the coprocessor requests, the granularity of the Follower Read load balancing policy is at the connection level. For a TiDB client connected to a specific Region, the selected follower is fixed, and is switched only when it fails or the scheduling policy is adjusted. -However, for the non-coprocessor requests, such as a point query, the granularity of the Follower Read load balancing policy is at the transaction level. For a TiDB transaction on a specific Region, the selected follower is fixed, and is switched only when it fails or the scheduling policy is adjusted. +However, for the non-coprocessor requests, such as a point query, the granularity of the Follower Read load balancing policy is at the transaction level. For a TiDB transaction on a specific Region, the selected follower is fixed, and is switched only when it fails or the scheduling policy is adjusted. If a transaction contains both point queries and coprocessor requests, the two types of requests are scheduled for reading separately according to the preceding scheduling policy. In this case, even if a coprocessor request and a point query are for the same Region, TiDB processes them as independent events. diff --git a/foreign-key.md b/foreign-key.md index 157d7e6d035b8..377a39ce065c5 100644 --- a/foreign-key.md +++ b/foreign-key.md @@ -10,7 +10,7 @@ Starting from v6.6.0, TiDB supports the foreign key feature, which allows cross- > **Warning:** > > - Currently, the foreign key feature is experimental. It is not recommended that you use it in production environments. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. -> - The foreign key feature is usually used for providing integrity and consistency constraint checks for data in small or medium volumes. However, for large data volumes in a distributed database system, the use of foreign keys might lead to serious performance issues and could have unpredictable effects on the system. If you plan to use foreign keys, conduct thorough validation first and use them with caution. +> - The foreign key feature is typically employed to enforce [referential integrity](https://en.wikipedia.org/wiki/Referential_integrity) constraint checks. It might cause performance degradation, so it is recommended to conduct thorough testing before using it in performance-sensitive scenarios. The foreign key is defined in the child table. The syntax is as follows: @@ -305,10 +305,10 @@ Create Table | CREATE TABLE `child` ( - [TiDB Binlog](/tidb-binlog/tidb-binlog-overview.md) does not support foreign keys. -- [DM](/dm/dm-overview.md) does not support foreign keys. DM v6.6.0 disables the [`foreign_key_checks`](/system-variables.md#foreign_key_checks) of the downstream TiDB when replicating data to TiDB. Therefore, the cascading operations caused by foreign keys are not replicated from the upstream to the downstream, which might cause data inconsistency. This behavior is consistent with the previous DM versions. +- [DM](/dm/dm-overview.md) does not support foreign keys. DM disables the [`foreign_key_checks`](/system-variables.md#foreign_key_checks) of the downstream TiDB when replicating data to TiDB. Therefore, the cascading operations caused by foreign keys are not replicated from the upstream to the downstream, which might cause data inconsistency. - [TiCDC](/ticdc/ticdc-overview.md) v6.6.0 is compatible with foreign keys. The previous versions of TiCDC might report an error when replicating tables with foreign keys. It is recommended to disable the `foreign_key_checks` of the downstream TiDB cluster when using a TiCDC version earlier than v6.6.0. - [BR](/br/backup-and-restore-overview.md) v6.6.0 is compatible with foreign keys. The previous versions of BR might report an error when restoring tables with foreign keys to a v6.6.0 or later cluster. It is recommended to disable the `foreign_key_checks` of the downstream TiDB cluster before restoring the cluster when using a BR earlier than v6.6.0. -- When you use [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md), it is recommended to disable the `foreign_key_checks` of the downstream TiDB cluster before importing data. +- When you use [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md), if the target table uses a foreign key, it is recommended to disable the `foreign_key_checks` of the downstream TiDB cluster before importing data. For versions earlier than v6.6.0, disabling this system variable does not take effect, and you need to grant the `REFERENCES` privilege for the downstream database user, or manually create the target table in the downstream database in advance to ensure smooth data import. diff --git a/functions-and-operators/aggregate-group-by-functions.md b/functions-and-operators/aggregate-group-by-functions.md index 2693c910f6ee9..f286d72359a8d 100644 --- a/functions-and-operators/aggregate-group-by-functions.md +++ b/functions-and-operators/aggregate-group-by-functions.md @@ -1,7 +1,6 @@ --- title: Aggregate (GROUP BY) Functions summary: Learn about the supported aggregate functions in TiDB. -aliases: ['/docs/dev/functions-and-operators/aggregate-group-by-functions/','/docs/dev/reference/sql/functions-and-operators/aggregate-group-by-functions/'] --- # Aggregate (GROUP BY) Functions @@ -63,7 +62,33 @@ In addition, TiDB also provides the following aggregate functions: 1 row in set (0.00 sec) ``` -Except for the `GROUP_CONCAT()` and `APPROX_PERCENTILE()` functions, all the preceding functions can serve as [Window functions](/functions-and-operators/window-functions.md). ++ `APPROX_COUNT_DISTINCT(expr, [expr...])` + + This function is similar to `COUNT(DISTINCT)` in counting the number of distinct values but returns an approximate result. It uses the `BJKST` algorithm, significantly reducing memory consumption when processing large datasets with a power-law distribution. Moreover, for low-cardinality data, this function provides high accuracy while maintaining efficient CPU utilization. + + The following example shows how to use this function: + + ```sql + DROP TABLE IF EXISTS t; + CREATE TABLE t(a INT, b INT, c INT); + INSERT INTO t VALUES(1, 1, 1), (2, 1, 1), (2, 2, 1), (3, 1, 1), (5, 1, 2), (5, 1, 2), (6, 1, 2), (7, 1, 2); + ``` + + ```sql + SELECT APPROX_COUNT_DISTINCT(a, b) FROM t GROUP BY c; + ``` + + ``` + +-----------------------------+ + | approx_count_distinct(a, b) | + +-----------------------------+ + | 3 | + | 4 | + +-----------------------------+ + 2 rows in set (0.00 sec) + ``` + +Except for the `GROUP_CONCAT()`, `APPROX_PERCENTILE()`, and `APPROX_COUNT_DISTINCT` functions, all the preceding functions can serve as [Window functions](/functions-and-operators/window-functions.md). ## GROUP BY modifiers diff --git a/functions-and-operators/bit-functions-and-operators.md b/functions-and-operators/bit-functions-and-operators.md index 1430d63bcdb6b..44066d7681c0b 100644 --- a/functions-and-operators/bit-functions-and-operators.md +++ b/functions-and-operators/bit-functions-and-operators.md @@ -1,7 +1,6 @@ --- title: Bit Functions and Operators summary: Learn about the bit functions and operators. -aliases: ['/docs/dev/functions-and-operators/bit-functions-and-operators/','/docs/dev/reference/sql/functions-and-operators/bit-functions-and-operators/'] --- # Bit Functions and Operators diff --git a/functions-and-operators/cast-functions-and-operators.md b/functions-and-operators/cast-functions-and-operators.md index 3eb512f074f69..063d774d64ea2 100644 --- a/functions-and-operators/cast-functions-and-operators.md +++ b/functions-and-operators/cast-functions-and-operators.md @@ -1,7 +1,6 @@ --- title: Cast Functions and Operators summary: Learn about the cast functions and operators. -aliases: ['/docs/dev/functions-and-operators/cast-functions-and-operators/','/docs/dev/reference/sql/functions-and-operators/cast-functions-and-operators/'] --- # Cast Functions and Operators diff --git a/functions-and-operators/control-flow-functions.md b/functions-and-operators/control-flow-functions.md index 7499c88f4b97f..786f62caa8b81 100644 --- a/functions-and-operators/control-flow-functions.md +++ b/functions-and-operators/control-flow-functions.md @@ -1,7 +1,6 @@ --- title: Control Flow Functions summary: Learn about the Control Flow functions. -aliases: ['/docs/dev/functions-and-operators/control-flow-functions/','/docs/dev/reference/sql/functions-and-operators/control-flow-functions/'] --- # Control Flow Functions diff --git a/functions-and-operators/date-and-time-functions.md b/functions-and-operators/date-and-time-functions.md index dd3568582c71c..be897e6abab89 100644 --- a/functions-and-operators/date-and-time-functions.md +++ b/functions-and-operators/date-and-time-functions.md @@ -1,7 +1,6 @@ --- title: Date and Time Functions summary: Learn how to use the data and time functions. -aliases: ['/docs/dev/functions-and-operators/date-and-time-functions/','/docs/dev/reference/sql/functions-and-operators/date-and-time-functions/'] --- # Date and Time Functions diff --git a/functions-and-operators/encryption-and-compression-functions.md b/functions-and-operators/encryption-and-compression-functions.md index 181300d313346..1116a97cec0ce 100644 --- a/functions-and-operators/encryption-and-compression-functions.md +++ b/functions-and-operators/encryption-and-compression-functions.md @@ -1,7 +1,6 @@ --- title: Encryption and Compression Functions summary: Learn about the encryption and compression functions. -aliases: ['/docs/dev/functions-and-operators/encryption-and-compression-functions/','/docs/dev/reference/sql/functions-and-operators/encryption-and-compression-functions/'] --- # Encryption and Compression Functions diff --git a/functions-and-operators/expressions-pushed-down.md b/functions-and-operators/expressions-pushed-down.md index bef6a53fa432f..763bdcc102fc3 100644 --- a/functions-and-operators/expressions-pushed-down.md +++ b/functions-and-operators/expressions-pushed-down.md @@ -1,14 +1,13 @@ --- title: List of Expressions for Pushdown summary: Learn a list of expressions that can be pushed down to TiKV and the related operations. -aliases: ['/docs/dev/functions-and-operators/expressions-pushed-down/','/docs/dev/reference/sql/functions-and-operators/expressions-pushed-down/'] --- # List of Expressions for Pushdown When TiDB reads data from TiKV, TiDB tries to push down some expressions (including calculations of functions or operators) to be processed to TiKV. This reduces the amount of transferred data and offloads processing from a single TiDB node. This document introduces the expressions that TiDB already supports pushing down and how to prohibit specific expressions from being pushed down using blocklist. -Tiflash also supports pushdown for the functions and operators [listed on this page](/tiflash/tiflash-supported-pushdown-calculations.md). +TiFlash also supports pushdown for the functions and operators [listed on this page](/tiflash/tiflash-supported-pushdown-calculations.md). ## Supported expressions for pushdown to TiKV @@ -19,9 +18,9 @@ Tiflash also supports pushdown for the functions and operators [listed on this p | [Comparison functions and operators](/functions-and-operators/operators.md#comparison-functions-and-operators) | [`<`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_less-than), [`<=`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_less-than-or-equal), [`=`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_equal), [`!= (<>)`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_not-equal), [`>`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_greater-than), [`>=`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_greater-than-or-equal), [`<=>`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_equal-to), [BETWEEN ... AND ...](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_between), [COALESCE()](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#function_coalesce), [IN()](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_in), [INTERVAL()](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#function_interval), [IS NOT NULL](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_is-not-null), [IS NOT](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_is-not), [IS NULL](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_is-null), [IS](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_is), [ISNULL()](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#function_isnull), [LIKE](https://dev.mysql.com/doc/refman/8.0/en/string-comparison-functions.html#operator_like), [NOT BETWEEN ... AND ...](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_not-between), [NOT IN()](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_not-in), [NOT LIKE](https://dev.mysql.com/doc/refman/8.0/en/string-comparison-functions.html#operator_not-like), [STRCMP()](https://dev.mysql.com/doc/refman/8.0/en/string-comparison-functions.html#function_strcmp) | | [Numeric functions and operators](/functions-and-operators/numeric-functions-and-operators.md) | [+](https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html#operator_plus), [-](https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html#operator_minus), [*](https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html#operator_times), [/](https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html#operator_divide), [DIV](https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html#operator_div), [% (MOD)](https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html#operator_mod), [-](https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html#operator_unary-minus), [ABS()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_abs), [ACOS()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_acos), [ASIN()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_asin), [ATAN()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_atan), [ATAN2(), ATAN()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_atan2), [CEIL()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_ceil), [CEILING()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_ceiling), [CONV()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_conv), [COS()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_cos), [COT()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_cot), [CRC32()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_crc32), [DEGREES()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_degrees), [EXP()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_exp), [FLOOR()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_floor), [LN()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_ln), [LOG()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_log), [LOG10()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_log10), [LOG2()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_log2), [MOD()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_mod), [PI()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_pi), [POW()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_pow), [POWER()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_power), [RADIANS()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_radians), [RAND()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_rand), [ROUND()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_round), [SIGN()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_sign), [SIN()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_sin), [SQRT()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_sqrt) | | [Control flow functions](/functions-and-operators/control-flow-functions.md) | [CASE](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#operator_case), [IF()](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#function_if), [IFNULL()](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#function_ifnull) | -| [JSON functions](/functions-and-operators/json-functions.md) | [JSON_ARRAY([val[, val] ...])](https://dev.mysql.com/doc/refman/8.0/en/json-creation-functions.html#function_json-array),
[JSON_CONTAINS(target, candidate[, path])](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-contains),
[JSON_EXTRACT(json_doc, path[, path] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-extract),
[JSON_INSERT(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-insert),
[JSON_LENGTH(json_doc[, path])](https://dev.mysql.com/doc/refman/8.0/en/json-attribute-functions.html#function_json-length),
[JSON_MERGE(json_doc, json_doc[, json_doc] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-merge),
[JSON_OBJECT([key, val[, key, val] ...])](https://dev.mysql.com/doc/refman/8.0/en/json-creation-functions.html#function_json-object),
[JSON_REMOVE(json_doc, path[, path] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-remove),
[JSON_REPLACE(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-replace),
[JSON_SET(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-set),
[JSON_TYPE(json_val)](https://dev.mysql.com/doc/refman/8.0/en/json-attribute-functions.html#function_json-type),
[JSON_UNQUOTE(json_val)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-unquote),
[JSON_VALID(val)](https://dev.mysql.com/doc/refman/8.0/en/json-attribute-functions.html#function_json-valid),
[value MEMBER OF(json_array)](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#operator_member-of) | +| [JSON functions](/functions-and-operators/json-functions.md) | [JSON_ARRAY([val[, val] ...])](https://dev.mysql.com/doc/refman/8.0/en/json-creation-functions.html#function_json-array),
[JSON_CONTAINS(target, candidate[, path])](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-contains),
[JSON_EXTRACT(json_doc, path[, path] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-extract),
[JSON_INSERT(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-insert),
[JSON_LENGTH(json_doc[, path])](https://dev.mysql.com/doc/refman/8.0/en/json-attribute-functions.html#function_json-length),
[JSON_MERGE(json_doc, json_doc[, json_doc] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-merge),
[JSON_OBJECT([key, val[, key, val] ...])](https://dev.mysql.com/doc/refman/8.0/en/json-creation-functions.html#function_json-object),
[JSON_REMOVE(json_doc, path[, path] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-remove),
[JSON_SET(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-set),
[JSON_TYPE(json_val)](https://dev.mysql.com/doc/refman/8.0/en/json-attribute-functions.html#function_json-type),
[JSON_UNQUOTE(json_val)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-unquote),
[JSON_VALID(val)](https://dev.mysql.com/doc/refman/8.0/en/json-attribute-functions.html#function_json-valid),
[value MEMBER OF(json_array)](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#operator_member-of) | | [Date and time functions](/functions-and-operators/date-and-time-functions.md) | [DATE()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_date), [DATE_FORMAT()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_date-format), [DATEDIFF()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_datediff), [DAYOFMONTH()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_dayofmonth), [DAYOFWEEK()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_dayofweek), [DAYOFYEAR()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_dayofyear), [FROM_DAYS()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_from-days), [HOUR()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_hour), [MAKEDATE()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_makedate), [MAKETIME()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_maketime), [MICROSECOND()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_microsecond), [MINUTE()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_minute), [MONTH()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_month), [MONTHNAME()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_monthname), [PERIOD_ADD()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_period-add), [PERIOD_DIFF()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_period-diff), [SEC_TO_TIME()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_sec-to-time), [SECOND()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_second), [SYSDATE()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_sysdate), [TIME_TO_SEC()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_time-to-sec), [TIMEDIFF()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_timediff), [WEEK()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_week), [WEEKOFYEAR()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_weekofyear), [YEAR()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_year) | -| [String functions](/functions-and-operators/string-functions.md) | [ASCII()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_ascii), [BIT_LENGTH()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_bit-length), [CHAR()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_char), [CHAR_LENGTH()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_char-length), [CONCAT()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_concat), [CONCAT_WS()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_concat-ws), [ELT()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_elt), [FIELD()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_field), [HEX()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_hex), [LENGTH()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_length), [LIKE](https://dev.mysql.com/doc/refman/8.0/en/string-comparison-functions.html#operator_like), [LTRIM()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_ltrim), [MID()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_mid), [NOT LIKE](https://dev.mysql.com/doc/refman/8.0/en/string-comparison-functions.html#operator_not-like), [NOT REGEXP](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#operator_not-regexp), [REGEXP](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#operator_regexp), [REPLACE()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_replace), [REVERSE()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_reverse), [RIGHT()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_right), [RTRIM()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_rtrim), [SPACE()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_space), [STRCMP()](https://dev.mysql.com/doc/refman/8.0/en/string-comparison-functions.html#function_strcmp), [SUBSTR()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_substr), [SUBSTRING()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_substring) | +| [String functions](/functions-and-operators/string-functions.md) | [ASCII()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_ascii), [BIT_LENGTH()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_bit-length), [CHAR()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_char), [CHAR_LENGTH()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_char-length), [CONCAT()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_concat), [CONCAT_WS()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_concat-ws), [ELT()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_elt), [FIELD()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_field), [HEX()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_hex), [LENGTH()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_length), [LIKE](https://dev.mysql.com/doc/refman/8.0/en/string-comparison-functions.html#operator_like), [LTRIM()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_ltrim), [MID()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_mid), [NOT LIKE](https://dev.mysql.com/doc/refman/8.0/en/string-comparison-functions.html#operator_not-like), [NOT REGEXP](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#operator_not-regexp), [REGEXP](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#operator_regexp), [REGEXP_INSTR()](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#function_regexp-instr), [REGEXP_LIKE()](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#function_regexp-like), [REGEXP_REPLACE()](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#function_regexp-replace), [REGEXP_SUBSTR()](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#function_regexp-substr), [REPLACE()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_replace), [REVERSE()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_reverse), [RIGHT()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_right), [RLIKE](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#operator_regexp), [RTRIM()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_rtrim), [SPACE()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_space), [STRCMP()](https://dev.mysql.com/doc/refman/8.0/en/string-comparison-functions.html#function_strcmp), [SUBSTR()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_substr), [SUBSTRING()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_substring) | | [Aggregation functions](/functions-and-operators/aggregate-group-by-functions.md#aggregate-group-by-functions) | [COUNT()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_count), [COUNT(DISTINCT)](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_count-distinct), [SUM()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_sum), [AVG()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_avg), [MAX()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_max), [MIN()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_min), [VARIANCE()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_variance), [VAR_POP()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_var-pop), [STD()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_std), [STDDEV()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_stddev), [STDDEV_POP](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_stddev-pop), [VAR_SAMP()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_var-samp), [STDDEV_SAMP()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_stddev-samp), [JSON_ARRAYAGG(key)](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_json-arrayagg), [JSON_OBJECTAGG(key, value)](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_json-objectagg) | | [Encryption and compression functions](/functions-and-operators/encryption-and-compression-functions.md#encryption-and-compression-functions) | [MD5()](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_md5), [SHA1(), SHA()](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_sha1), [UNCOMPRESSED_LENGTH()](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_uncompressed-length) | | [Cast functions and operators](/functions-and-operators/cast-functions-and-operators.md#cast-functions-and-operators) | [CAST()](https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html#function_cast), [CONVERT()](https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html#function_convert) | diff --git a/functions-and-operators/functions-and-operators-overview.md b/functions-and-operators/functions-and-operators-overview.md index ec1253087ea5e..a993eaa3675a0 100644 --- a/functions-and-operators/functions-and-operators-overview.md +++ b/functions-and-operators/functions-and-operators-overview.md @@ -1,7 +1,6 @@ --- title: Function and Operator Reference summary: Learn how to use the functions and operators. -aliases: ['/docs/dev/functions-and-operators/functions-and-operators-overview/','/docs/dev/reference/sql/functions-and-operators/reference/'] --- # Function and Operator Reference diff --git a/functions-and-operators/information-functions.md b/functions-and-operators/information-functions.md index 2f88e11d70972..788e00b8c4e72 100644 --- a/functions-and-operators/information-functions.md +++ b/functions-and-operators/information-functions.md @@ -1,7 +1,6 @@ --- title: Information Functions summary: Learn about the information functions. -aliases: ['/docs/dev/functions-and-operators/information-functions/','/docs/dev/reference/sql/functions-and-operators/information-functions/'] --- # Information Functions diff --git a/functions-and-operators/json-functions.md b/functions-and-operators/json-functions.md index 6763335887590..059a79086929e 100644 --- a/functions-and-operators/json-functions.md +++ b/functions-and-operators/json-functions.md @@ -1,7 +1,6 @@ --- title: JSON Functions summary: Learn about JSON functions. -aliases: ['/docs/dev/functions-and-operators/json-functions/','/docs/dev/reference/sql/functions-and-operators/json-functions/'] --- # JSON Functions diff --git a/functions-and-operators/miscellaneous-functions.md b/functions-and-operators/miscellaneous-functions.md index e0f3e73f0d9b4..0c8ea6636d00b 100644 --- a/functions-and-operators/miscellaneous-functions.md +++ b/functions-and-operators/miscellaneous-functions.md @@ -1,7 +1,6 @@ --- title: Miscellaneous Functions summary: Learn about miscellaneous functions in TiDB. -aliases: ['/docs/dev/functions-and-operators/miscellaneous-functions/','/docs/dev/reference/sql/functions-and-operators/miscellaneous-functions/'] --- # Miscellaneous Functions @@ -24,7 +23,7 @@ TiDB supports most of the [miscellaneous functions](https://dev.mysql.com/doc/re | [`IS_IPV4_MAPPED()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_is-ipv4-mapped) | Whether argument is an IPv4-mapped address | | [`IS_IPV6()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_is-ipv6) | Whether argument is an IPv6 address | | [`NAME_CONST()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_name-const) | Can be used to rename a column name | -| [`SLEEP()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_sleep) | Sleep for a number of seconds. Note that for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters, the `SLEEP()` function has a limitation wherein it can only support a maximum sleep time of 300 seconds. | +| [`SLEEP()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_sleep) | Sleep for a number of seconds. Note that for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters, the `SLEEP()` function has a limitation wherein it can only support a maximum sleep time of 300 seconds. | | [`UUID()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_uuid) | Return a Universal Unique Identifier (UUID) | | [`UUID_TO_BIN()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_uuid-to-bin) | Convert UUID from text format to binary format | | [`VALUES()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_values) | Defines the values to be used during an INSERT | diff --git a/functions-and-operators/numeric-functions-and-operators.md b/functions-and-operators/numeric-functions-and-operators.md index ab228e0f1ea40..c761b2207d4c4 100644 --- a/functions-and-operators/numeric-functions-and-operators.md +++ b/functions-and-operators/numeric-functions-and-operators.md @@ -1,7 +1,6 @@ --- title: Numeric Functions and Operators summary: Learn about the numeric functions and operators. -aliases: ['/docs/dev/functions-and-operators/numeric-functions-and-operators/','/docs/dev/reference/sql/functions-and-operators/numeric-functions-and-operators/'] --- # Numeric Functions and Operators diff --git a/functions-and-operators/operators.md b/functions-and-operators/operators.md index 665e30961135c..04c69dc98f25e 100644 --- a/functions-and-operators/operators.md +++ b/functions-and-operators/operators.md @@ -1,7 +1,6 @@ --- title: Operators summary: Learn about the operators precedence, comparison functions and operators, logical operators, and assignment operators. -aliases: ['/docs/dev/functions-and-operators/operators/','/docs/dev/reference/sql/functions-and-operators/operators/'] --- # Operators diff --git a/functions-and-operators/precision-math.md b/functions-and-operators/precision-math.md index fca0983e8ab41..dbabf4d122b29 100644 --- a/functions-and-operators/precision-math.md +++ b/functions-and-operators/precision-math.md @@ -1,7 +1,6 @@ --- title: Precision Math summary: Learn about the precision math in TiDB. -aliases: ['/docs/dev/functions-and-operators/precision-math/','/docs/dev/reference/sql/functions-and-operators/precision-math/'] --- # Precision Math @@ -51,7 +50,7 @@ DECIMAL columns do not store a leading `+` character or `-` character or leading DECIMAL columns do not permit values larger than the range implied by the column definition. For example, a `DECIMAL(3,0)` column supports a range of `-999` to `999`. A `DECIMAL(M,D)` column permits at most `M - D` digits to the left of the decimal point. -For more information about the internal format of the DECIMAL values, see [`mydecimal.go`](https://github.com/pingcap/tidb/blob/master/pkg/types/mydecimal.go) in TiDB souce code. +For more information about the internal format of the DECIMAL values, see [`mydecimal.go`](https://github.com/pingcap/tidb/blob/release-7.5/pkg/types/mydecimal.go) in TiDB source code. ## Expression handling diff --git a/functions-and-operators/sequence-functions.md b/functions-and-operators/sequence-functions.md new file mode 100644 index 0000000000000..bbf14b6b2e624 --- /dev/null +++ b/functions-and-operators/sequence-functions.md @@ -0,0 +1,18 @@ +--- +title: Sequence Functions +summary: This document introduces sequence functions supported in TiDB. +--- + +# Sequence Functions + +Sequence functions in TiDB are used to return or set values of sequence objects created using the [`CREATE SEQUENCE`](/sql-statements/sql-statement-create-sequence.md) statement. + +| Function name | Feature description | +| :-------------- | :------------------------------------- | +| `NEXTVAL()` or `NEXT VALUE FOR` | Returns the next value of a sequence | +| `SETVAL()` | Sets the current value of a sequence | +| `LASTVAL()` | Returns the last used value of a sequence | + +## MySQL compatibility + +MySQL does not support the functions and statements for creating and manipulating sequences as defined in [ISO/IEC 9075-2](https://www.iso.org/standard/76584.html). diff --git a/functions-and-operators/string-functions.md b/functions-and-operators/string-functions.md index d48dca7055887..eab1b5b96cbdf 100644 --- a/functions-and-operators/string-functions.md +++ b/functions-and-operators/string-functions.md @@ -1,7 +1,6 @@ --- title: String Functions summary: Learn about the string functions in TiDB. -aliases: ['/docs/dev/functions-and-operators/string-functions/','/docs/dev/reference/sql/functions-and-operators/string-functions/'] --- # String Functions @@ -85,7 +84,7 @@ For comparisons between functions and syntax of Oracle and TiDB, see [Comparison ## Regular expression compatibility with MySQL -The following sections describe the regular expression compatibility with MySQL. +The following sections describe the regular expression compatibility with MySQL, including `REGEXP_INSTR()`, `REGEXP_LIKE()`, `REGEXP_REPLACE()`, and `REGEXP_SUBSTR()`. ### Syntax compatibility @@ -111,9 +110,15 @@ The difference between TiDB and MySQL support for the binary string type: ### Other compatibility -The difference between TiDB and MySQL support in replacing empty strings: +- The behavior of replacing empty strings in TiDB is different from MySQL. Taking `REGEXP_REPLACE("", "^$", "123")` as an example: -The following takes `REGEXP_REPLACE("", "^$", "123")` as an example: + - MySQL does not replace the empty string and returns `""` as the result. + - TiDB replaces the empty string and returns `"123"` as the result. -- MySQL does not replace the empty string and returns `""` as the result. -- TiDB replaces the empty string and returns `"123"` as the result. +- The keyword used for capturing groups in TiDB is different from MySQL. MySQL uses `$` as the keyword, while TiDB uses `\\` as the keyword. In addition, TiDB only supports capturing groups numbered from `0` to `9`. + + For example, the following SQL statement returns `ab` in TiDB: + + ```sql + SELECT REGEXP_REPLACE('abcd','(.*)(.{2})$','\\1') AS s; + ``` diff --git a/functions-and-operators/tidb-functions.md b/functions-and-operators/tidb-functions.md index c853d5b632c89..c28a7b2789440 100644 --- a/functions-and-operators/tidb-functions.md +++ b/functions-and-operators/tidb-functions.md @@ -11,18 +11,21 @@ The following functions are TiDB extensions, and are not present in MySQL: | Function name | Function description | | :-------------- | :------------------------------------- | -| `TIDB_BOUNDED_STALENESS()` | The `TIDB_BOUNDED_STALENESS` function instructs TiDB to read the data as new as possible within the time range. See also: [Read Historical Data Using the `AS OF TIMESTAMP` Clause](/as-of-timestamp.md) | -| [`TIDB_DECODE_KEY(str)`](#tidb_decode_key) | The `TIDB_DECODE_KEY` function can be used to decode a TiDB-encoded key entry into a JSON structure containing `_tidb_rowid` and `table_id`. These encoded keys can be found in some system tables and in logging outputs. | -| [`TIDB_DECODE_PLAN(str)`](#tidb_decode_plan) | The `TIDB_DECODE_PLAN` function can be used to decode a TiDB execution plan. | -| `TIDB_IS_DDL_OWNER()` | The `TIDB_IS_DDL_OWNER` function can be used to check whether or not the TiDB instance you are connected to is the one that is the DDL Owner. The DDL Owner is the TiDB instance that is tasked with executing DDL statements on behalf of all other nodes in the cluster. | -| [`TIDB_PARSE_TSO(num)`](#tidb_parse_tso) | The `TIDB_PARSE_TSO` function can be used to extract the physical timestamp from a TiDB TSO timestamp. See also: [`tidb_current_ts`](/system-variables.md#tidb_current_ts). | -| `TIDB_PARSE_TSO_LOGICAL(num)` | The `TIDB_PARSE_TSO_LOGICAL` function can be used to extract the logical timestamp from a TiDB TSO timestamp. | -| [`TIDB_VERSION()`](#tidb_version) | The `TIDB_VERSION` function returns the TiDB version with additional build information. | -| [`TIDB_DECODE_SQL_DIGESTS(digests, stmtTruncateLength)`](#tidb_decode_sql_digests) | The `TIDB_DECODE_SQL_DIGESTS()` function is used to query the normalized SQL statements (a form without formats and arguments) corresponding to the set of SQL digests in the cluster. | -| `VITESS_HASH(str)` | The `VITESS_HASH` function returns the hash of a string that is compatible with Vitess' `HASH` function. This is intended to help the data migration from Vitess. | -| `TIDB_SHARD()` | The `TIDB_SHARD` function can be used to create a shard index to scatter the index hotspot. A shard index is an expression index with a `TIDB_SHARD` function as the prefix.| -| `TIDB_ROW_CHECKSUM()` | The `TIDB_ROW_CHECKSUM` function is used to query the checksum value of a row. This function can only be used in `SELECT` statements within the FastPlan process. That is, you can query through statements like `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id = ?` or `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id IN (?, ?, ...)`. See also: [Data integrity validation for single-row data](/ticdc/ticdc-integrity-check.md). | -| `CURRENT_RESOURCE_GROUP()` | The `CURRENT_RESOURCE_GROUP` function is used to return the resource group name that the current session is bound to. See also: [Use Resource Control to Achieve Resource Isolation](/tidb-resource-control.md). | +| [`CURRENT_RESOURCE_GROUP()`](#current_resource_group) | Returns the resource group name that the current session is bound to. See also: [Use Resource Control to Achieve Resource Isolation](/tidb-resource-control.md). | +| [`TIDB_BOUNDED_STALENESS()`](#tidb_bounded_staleness) | Instructs TiDB to read the data as new as possible within the time range. See also: [Read Historical Data Using the `AS OF TIMESTAMP` Clause](/as-of-timestamp.md). | +| [`TIDB_CURRENT_TSO()`](#tidb_current_tso) | Returns the current [TSO](/tso.md). | +| [`TIDB_DECODE_BINARY_PLAN()`](#tidb_decode_binary_plan) | Decodes binary plans. | +| [`TIDB_DECODE_KEY()`](#tidb_decode_key) | Decodes a TiDB-encoded key entry into a JSON structure containing `_tidb_rowid` and `table_id`. These encoded keys can be found in some system tables and logging outputs. | +| [`TIDB_DECODE_PLAN()`](#tidb_decode_plan) | Decodes a TiDB execution plan. | +| [`TIDB_DECODE_SQL_DIGESTS()`](#tidb_decode_sql_digests) | Queries the normalized SQL statements (a form without formats and arguments) corresponding to the set of SQL digests in the cluster. | +| [`TIDB_ENCODE_SQL_DIGEST()`](#tidb_encode_sql_digest) | Gets a digest for a query string. | +| [`TIDB_IS_DDL_OWNER()`](#tidb_is_ddl_owner) | Checks whether or not the TiDB instance you are connected to is the one that is the DDL Owner. The DDL Owner is the TiDB instance that is tasked with executing DDL statements on behalf of all other nodes in the cluster. | +| [`TIDB_PARSE_TSO()`](#tidb_parse_tso) | Extracts the physical timestamp from a TiDB TSO timestamp. See also: [`tidb_current_ts`](/system-variables.md#tidb_current_ts). | +| [`TIDB_PARSE_TSO_LOGICAL()`](#tidb_parse_tso_logical) | Extracts the logical timestamp from a TiDB TSO timestamp. | +| [`TIDB_ROW_CHECKSUM()`](#tidb_row_checksum) | Queries the checksum value of a row. This function can only be used in `SELECT` statements within the FastPlan process. That is, you can query through statements like `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id = ?` or `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id IN (?, ?, ...)`. See also: [Data integrity validation for single-row data](/ticdc/ticdc-integrity-check.md). | +| [`TIDB_SHARD()`](#tidb_shard) | Creates a shard index to scatter the index hotspot. A shard index is an expression index with a `TIDB_SHARD` function as the prefix.| +| [`TIDB_VERSION()`](#tidb_version) | Returns the TiDB version with additional build information. | +| [`VITESS_HASH()`](#vitess_hash) | Returns the hash of a number that is compatible with Vitess' `HASH` function. This is intended to help the data migration from Vitess. |
@@ -30,30 +33,142 @@ The following functions are TiDB extensions, and are not present in MySQL: | Function name | Function description | | :-------------- | :------------------------------------- | -| `TIDB_BOUNDED_STALENESS()` | The `TIDB_BOUNDED_STALENESS` function instructs TiDB to read the data as new as possible within the time range. See also: [Read Historical Data Using the `AS OF TIMESTAMP` Clause](/as-of-timestamp.md) | -| [`TIDB_DECODE_KEY(str)`](#tidb_decode_key) | The `TIDB_DECODE_KEY` function can be used to decode a TiDB-encoded key entry into a JSON structure containing `_tidb_rowid` and `table_id`. These encoded keys can be found in some system tables and in logging outputs. | -| [`TIDB_DECODE_PLAN(str)`](#tidb_decode_plan) | The `TIDB_DECODE_PLAN` function can be used to decode a TiDB execution plan. | -| `TIDB_IS_DDL_OWNER()` | The `TIDB_IS_DDL_OWNER` function can be used to check whether or not the TiDB instance you are connected to is the one that is the DDL Owner. The DDL Owner is the TiDB instance that is tasked with executing DDL statements on behalf of all other nodes in the cluster. | -| [`TIDB_PARSE_TSO(num)`](#tidb_parse_tso) | The `TIDB_PARSE_TSO` function can be used to extract the physical timestamp from a TiDB TSO timestamp. See also: [`tidb_current_ts`](/system-variables.md#tidb_current_ts). | -| `TIDB_PARSE_TSO_LOGICAL(num)` | The `TIDB_PARSE_TSO_LOGICAL` function can be used to extract the logical timestamp from a TiDB TSO timestamp. | -| [`TIDB_VERSION()`](#tidb_version) | The `TIDB_VERSION` function returns the TiDB version with additional build information. | -| [`TIDB_DECODE_SQL_DIGESTS(digests, stmtTruncateLength)`](#tidb_decode_sql_digests) | The `TIDB_DECODE_SQL_DIGESTS()` function is used to query the normalized SQL statements (a form without formats and arguments) corresponding to the set of SQL digests in the cluster. | -| `VITESS_HASH(str)` | The `VITESS_HASH` function returns the hash of a string that is compatible with Vitess' `HASH` function. This is intended to help the data migration from Vitess. | -| `TIDB_SHARD()` | The `TIDB_SHARD` function can be used to create a shard index to scatter the index hotspot. A shard index is an expression index with a `TIDB_SHARD` function as the prefix.| -| `TIDB_ROW_CHECKSUM()` | The `TIDB_ROW_CHECKSUM` function is used to query the checksum value of a row. This function can only be used in `SELECT` statements within the FastPlan process. That is, you can query through statements like `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id = ?` or `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id IN (?, ?, ...)`. See also: [Data integrity validation for single-row data](https://docs.pingcap.com/tidb/stable/ticdc-integrity-check). | -| `CURRENT_RESOURCE_GROUP()` | The `CURRENT_RESOURCE_GROUP` function is used to return the resource group name that the current session is bound to. See also: [Use Resource Control to Achieve Resource Isolation](/tidb-resource-control.md). | +| [`CURRENT_RESOURCE_GROUP()`](#current_resource_group) | Returns the resource group name that the current session is bound to. See also: [Use Resource Control to Achieve Resource Isolation](/tidb-resource-control.md). | +| [`TIDB_BOUNDED_STALENESS()`](#tidb_bounded_staleness) | Instructs TiDB to read the data as new as possible within the time range. See also: [Read Historical Data Using the `AS OF TIMESTAMP` Clause](/as-of-timestamp.md). | +| [`TIDB_CURRENT_TSO()`](#tidb_current_tso) | Returns the current [TSO](/tso.md). | +| [`TIDB_DECODE_BINARY_PLAN()`](#tidb_decode_binary_plan) | Decodes binary plans. | +| [`TIDB_DECODE_KEY()`](#tidb_decode_key) | Decodes a TiDB-encoded key entry into a JSON structure containing `_tidb_rowid` and `table_id`. These encoded keys can be found in some system tables and in logging outputs. | +| [`TIDB_DECODE_PLAN()`](#tidb_decode_plan) | Decodes a TiDB execution plan. | +| [`TIDB_DECODE_SQL_DIGESTS()`](#tidb_decode_sql_digests) | Queries the normalized SQL statements (a form without formats and arguments) corresponding to the set of SQL digests in the cluster. | +| [`TIDB_ENCODE_SQL_DIGEST()`](#tidb_encode_sql_digest) | Gets a digest for a query string. | +| [`TIDB_IS_DDL_OWNER()`](#tidb_is_ddl_owner) | Checks whether or not the TiDB instance you are connected to is the one that is the DDL Owner. The DDL Owner is the TiDB instance that is tasked with executing DDL statements on behalf of all other nodes in the cluster. | +| [`TIDB_PARSE_TSO()`](#tidb_parse_tso) | Extracts the physical timestamp from a TiDB TSO timestamp. See also: [`tidb_current_ts`](/system-variables.md#tidb_current_ts). | +| [`TIDB_PARSE_TSO_LOGICAL()`](#tidb_parse_tso_logical) | Extracts the logical timestamp from a TiDB TSO timestamp. | +| [`TIDB_ROW_CHECKSUM()`](#tidb_row_checksum) | Queries the checksum value of a row. This function can only be used in `SELECT` statements within the FastPlan process. That is, you can query through statements like `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id = ?` or `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id IN (?, ?, ...)`. See also: [Data integrity validation for single-row data](https://docs.pingcap.com/tidb/stable/ticdc-integrity-check). | +| [`TIDB_SHARD()`](#tidb_shard) | Creates a shard index to scatter the index hotspot. A shard index is an expression index with a `TIDB_SHARD` function as the prefix.| +| [`TIDB_VERSION()`](#tidb_version) | Returns the TiDB version with additional build information. | +| [`VITESS_HASH()`](#vitess_hash) | Returns the hash of a number that is compatible with Vitess' `HASH` function. This is intended to help the data migration from Vitess. | -## Examples +## CURRENT_RESOURCE_GROUP -This section provides examples for some of the functions above. +The `CURRENT_RESOURCE_GROUP` function is used to show the resource group name that the current session is bound to. When the [Resource control](/tidb-resource-control.md) feature is enabled, the available resources that can be used by SQL statements are restricted by the resource quota of the bound resource group. + +When a session is established, TiDB binds the session to the resource group that the login user is bound to by default. If the user is not bound to any resource groups, the session is bound to the `default` resource group. Once the session is established, the bound resource group will not change by default, even if the bound resource group of the user is changed via [modifying the resource group bound to the user](/sql-statements/sql-statement-alter-user.md#modify-basic-user-information). To change the bound resource group of the current session, you can use [`SET RESOURCE GROUP`](/sql-statements/sql-statement-set-resource-group.md). + +Examples: + +Create a user `user1`, create two resource groups `rg1` and `rg2`, and bind the user `user1` to the resource group `rg1`: + +```sql +CREATE USER 'user1'; +CREATE RESOURCE GROUP 'rg1' RU_PER_SEC = 1000; +CREATE RESOURCE GROUP 'rg2' RU_PER_SEC = 2000; +ALTER USER 'user1' RESOURCE GROUP `rg1`; +``` + +Use `user1` to log in and view the resource group bound to the current user: + +```sql +SELECT CURRENT_RESOURCE_GROUP(); +``` + +``` ++--------------------------+ +| CURRENT_RESOURCE_GROUP() | ++--------------------------+ +| rg1 | ++--------------------------+ +1 row in set (0.00 sec) +``` + +Execute `SET RESOURCE GROUP` to set the resource group for the current session to `rg2`, and then view the resource group bound to the current user: + +```sql +SET RESOURCE GROUP `rg2`; +SELECT CURRENT_RESOURCE_GROUP(); +``` + +``` ++--------------------------+ +| CURRENT_RESOURCE_GROUP() | ++--------------------------+ +| rg2 | ++--------------------------+ +1 row in set (0.00 sec) +``` + +## TIDB_BOUNDED_STALENESS + +The `TIDB_BOUNDED_STALENESS()` function is used as part of [`AS OF TIMESTAMP`](/as-of-timestamp.md) syntax. + +## TIDB_CURRENT_TSO + +The `TIDB_CURRENT_TSO()` function returns the [TSO](/tso.md) for the current transaction. This is similar to the [`tidb_current_ts`](/system-variables.md#tidb_current_ts) variable. + +```sql +BEGIN; +``` + +``` +Query OK, 0 rows affected (0.00 sec) +``` + +```sql +SELECT TIDB_CURRENT_TSO(); +``` + +``` ++--------------------+ +| TIDB_CURRENT_TSO() | ++--------------------+ +| 450456244814610433 | ++--------------------+ +1 row in set (0.00 sec) +``` + +```sql +SELECT @@tidb_current_ts; +``` + +``` ++--------------------+ +| @@tidb_current_ts | ++--------------------+ +| 450456244814610433 | ++--------------------+ +1 row in set (0.00 sec) +``` + +## TIDB_DECODE_BINARY_PLAN + +The `TIDB_DECODE_BINARY_PLAN(binary_plan)` function decodes binary plans, like the ones in the `BINARY_PLAN` column of the [`STATEMENTS_SUMMARY`](/statement-summary-tables.md) table. + +The [`tidb_generate_binary_plan`](/system-variables.md#tidb_generate_binary_plan-new-in-v620) variable must be set to `ON` for the binary plans to be available. + +Example: + +```sql +SELECT BINARY_PLAN,TIDB_DECODE_BINARY_PLAN(BINARY_PLAN) FROM information_schema.STATEMENTS_SUMMARY LIMIT 1\G +``` + +``` +*************************** 1. row *************************** + BINARY_PLAN: lQLwPgqQAgoMUHJvamVjdGlvbl8zEngKDk1lbVRhYmxlU2Nhbl80KQAAAAAAiMNAMAM4AUABSioKKAoSaW5mb3JtYQU00HNjaGVtYRISU1RBVEVNRU5UU19TVU1NQVJZWhV0aW1lOjI5LjPCtXMsIGxvb3BzOjJw////CQIEAXgJCBD///8BIQFnDOCb+EA6cQCQUjlDb2x1bW4jOTIsIHRpZGJfZGVjb2RlX2JpbmFyeV9wbGFuKBUjCCktPg0MEDEwM1oWBYAIMTA4NoEAeGINQ29uY3VycmVuY3k6NXDIZXj///////////8BGAE= +TIDB_DECODE_BINARY_PLAN(BINARY_PLAN): +| id | estRows | estCost | actRows | task | access object | execution info | operator info | memory | disk | +| Projection_3 | 10000.00 | 100798.00 | 3 | root | | time:108.3µs, loops:2, Concurrency:5 | Column#92, tidb_decode_binary_plan(Column#92)->Column#103 | 12.7 KB | N/A | +| └─MemTableScan_4 | 10000.00 | 0.00 | 3 | root | table:STATEMENTS_SUMMARY | time:29.3µs, loops:2 | | N/A | N/A | -### TIDB_DECODE_KEY +1 row in set (0.00 sec) +``` + +## TIDB_DECODE_KEY -In the following example, the table `t1` has a hidden `rowid` that is generated by TiDB. The `TIDB_DECODE_KEY` is used in the statement. From the result, you can see that the hidden `rowid` is decoded and output, which is a typical result for the non-clustered primary key. +The `TIDB_DECODE_KEY()` function decodes a TiDB-encoded key entry into a JSON structure containing `_tidb_rowid` and `table_id`. These encoded keys exist in some system tables and logging outputs. -{{< copyable "sql" >}} +In the following example, the table `t1` has a hidden `rowid` that is generated by TiDB. The `TIDB_DECODE_KEY()` function is used in the statement. From the result, you can see that the hidden `rowid` is decoded and output, which is a typical result for the non-clustered primary key. ```sql SELECT START_KEY, TIDB_DECODE_KEY(START_KEY) FROM information_schema.tikv_region_status WHERE table_name='t1' AND REGION_ID=2\G @@ -68,10 +183,8 @@ TIDB_DECODE_KEY(START_KEY): {"_tidb_rowid":1958897,"table_id":"59"} In the following example, the table `t2` has a compound clustered primary key. From the JSON output, you can see a `handle` that contains the name and value for both of the columns that are part of the primary key. -{{< copyable "sql" >}} - ```sql -show create table t2\G +SHOW CREATE TABLE t2\G ``` ```sql @@ -86,10 +199,8 @@ Create Table: CREATE TABLE `t2` ( 1 row in set (0.001 sec) ``` -{{< copyable "sql" >}} - ```sql -select * from information_schema.tikv_region_status where table_name='t2' limit 1\G +SELECT * FROM information_schema.tikv_region_status WHERE table_name='t2' LIMIT 1\G ``` ```sql @@ -114,10 +225,8 @@ REPLICATIONSTATUS_STATEID: NULL 1 row in set (0.005 sec) ``` -{{< copyable "sql" >}} - ```sql -select tidb_decode_key('7480000000000000FF3E5F720400000000FF0000000601633430FF3338646232FF2D64FF3531632D3131FF65FF622D386337352DFFFF3830653635303138FFFF61396265000000FF00FB000000000000F9'); +SELECT tidb_decode_key('7480000000000000FF3E5F720400000000FF0000000601633430FF3338646232FF2D64FF3531632D3131FF65FF622D386337352DFFFF3830653635303138FFFF61396265000000FF00FB000000000000F9'); ``` ```sql @@ -129,73 +238,55 @@ select tidb_decode_key('7480000000000000FF3E5F720400000000FF0000000601633430FF33 1 row in set (0.001 sec) ``` -### TIDB_DECODE_PLAN - -You can find TiDB execution plans in encoded form in the slow query log. The `TIDB_DECODE_PLAN()` function is then used to decode the encoded plans into a human-readable form. - -This function is useful because a plan is captured at the time the statement is executed. Re-executing the statement in `EXPLAIN` might produce different results as data distribution and statistics evolves over time. +In the following example, the first Region of a table starts with a key that only has the `table_id` of the table. The last Region of the table ends with `table_id + 1`. Any Regions in between have longer keys that includes a `_tidb_rowid` or `handle`. ```sql -SELECT tidb_decode_plan('8QIYMAkzMV83CQEH8E85LjA0CWRhdGE6U2VsZWN0aW9uXzYJOTYwCXRpbWU6NzEzLjHCtXMsIGxvb3BzOjIsIGNvcF90YXNrOiB7bnVtOiAxLCBtYXg6IDU2OC41wgErRHByb2Nfa2V5czogMCwgcnBjXxEpAQwFWBAgNTQ5LglZyGNvcHJfY2FjaGVfaGl0X3JhdGlvOiAwLjAwfQkzLjk5IEtCCU4vQQoxCTFfNgkxXzAJMwm2SGx0KHRlc3QudC5hLCAxMDAwMCkNuQRrdgmiAHsFbBQzMTMuOMIBmQnEDDk2MH0BUgEEGAoyCTQzXzUFVwX1oGFibGU6dCwga2VlcCBvcmRlcjpmYWxzZSwgc3RhdHM6cHNldWRvCTk2ISE2aAAIMTUzXmYA')\G +SELECT + TABLE_NAME, + TIDB_DECODE_KEY(START_KEY), + TIDB_DECODE_KEY(END_KEY) +FROM + information_schema.TIKV_REGION_STATUS +WHERE + TABLE_NAME='stock' + AND IS_INDEX=0 +ORDER BY + START_KEY; ``` ```sql -*************************** 1. row *************************** - tidb_decode_plan('8QIYMAkzMV83CQEH8E85LjA0CWRhdGE6U2VsZWN0aW9uXzYJOTYwCXRpbWU6NzEzLjHCtXMsIGxvb3BzOjIsIGNvcF90YXNrOiB7bnVtOiAxLCBtYXg6IDU2OC41wgErRHByb2Nfa2V5czogMCwgcnBjXxEpAQwFWBAgNTQ5LglZyGNvcHJfY2FjaGVfaGl0X3JhdGlvOiAwLjAwfQkzLjk5IEtCCU4vQQoxCTFfNgkxXz: id task estRows operator info actRows execution info memory disk - TableReader_7 root 319.04 data:Selection_6 960 time:713.1µs, loops:2, cop_task: {num: 1, max: 568.5µs, proc_keys: 0, rpc_num: 1, rpc_time: 549.1µs, copr_cache_hit_ratio: 0.00} 3.99 KB N/A - └─Selection_6 cop[tikv] 319.04 lt(test.t.a, 10000) 960 tikv_task:{time:313.8µs, loops:960} N/A N/A - └─TableFullScan_5 cop[tikv] 960 table:t, keep order:false, stats:pseudo 960 tikv_task:{time:153µs, loops:960} N/A N/A -``` - -### TIDB_PARSE_TSO - -The `TIDB_PARSE_TSO` function can be used to extract the physical timestamp from a TiDB TSO timestamp. TSO stands for Time Stamp Oracle and is a monotonically increasing timestamp given out by PD (Placement Driver) for every transaction. - -A TSO is a number that consists of two parts: - -- A physical timestamp -- A logical counter - -```sql -BEGIN; -SELECT TIDB_PARSE_TSO(@@tidb_current_ts); -ROLLBACK; ++------------+-----------------------------------------------------------+-----------------------------------------------------------+ +| TABLE_NAME | TIDB_DECODE_KEY(START_KEY) | TIDB_DECODE_KEY(END_KEY) | ++------------+-----------------------------------------------------------+-----------------------------------------------------------+ +| stock | {"table_id":143} | {"handle":{"s_i_id":"32485","s_w_id":"3"},"table_id":143} | +| stock | {"handle":{"s_i_id":"32485","s_w_id":"3"},"table_id":143} | {"handle":{"s_i_id":"64964","s_w_id":"5"},"table_id":143} | +| stock | {"handle":{"s_i_id":"64964","s_w_id":"5"},"table_id":143} | {"handle":{"s_i_id":"97451","s_w_id":"7"},"table_id":143} | +| stock | {"handle":{"s_i_id":"97451","s_w_id":"7"},"table_id":143} | {"table_id":145} | ++------------+-----------------------------------------------------------+-----------------------------------------------------------+ +4 rows in set (0.031 sec) ``` -```sql -+-----------------------------------+ -| TIDB_PARSE_TSO(@@tidb_current_ts) | -+-----------------------------------+ -| 2021-05-26 11:33:37.776000 | -+-----------------------------------+ -1 row in set (0.0012 sec) -``` +`TIDB_DECODE_KEY` returns valid JSON on success and returns the argument value if it fails to decode. -Here `TIDB_PARSE_TSO` is used to extract the physical timestamp from the timestamp number that is available in the `tidb_current_ts` session variable. Because timestamps are given out per transaction, this function is running in a transaction. +## TIDB_DECODE_PLAN -### TIDB_VERSION +You can find TiDB execution plans in encoded form in the slow query log. The `TIDB_DECODE_PLAN()` function is then used to decode the encoded plans into a human-readable form. -The `TIDB_VERSION` function can be used to get the version and build details of the TiDB server that you are connected to. You can use this function when reporting issues on GitHub. +This function is useful because a plan is captured at the time the statement is executed. Re-executing the statement in `EXPLAIN` might produce different results as data distribution and statistics evolves over time. ```sql -SELECT TIDB_VERSION()\G +SELECT tidb_decode_plan('8QIYMAkzMV83CQEH8E85LjA0CWRhdGE6U2VsZWN0aW9uXzYJOTYwCXRpbWU6NzEzLjHCtXMsIGxvb3BzOjIsIGNvcF90YXNrOiB7bnVtOiAxLCBtYXg6IDU2OC41wgErRHByb2Nfa2V5czogMCwgcnBjXxEpAQwFWBAgNTQ5LglZyGNvcHJfY2FjaGVfaGl0X3JhdGlvOiAwLjAwfQkzLjk5IEtCCU4vQQoxCTFfNgkxXzAJMwm2SGx0KHRlc3QudC5hLCAxMDAwMCkNuQRrdgmiAHsFbBQzMTMuOMIBmQnEDDk2MH0BUgEEGAoyCTQzXzUFVwX1oGFibGU6dCwga2VlcCBvcmRlcjpmYWxzZSwgc3RhdHM6cHNldWRvCTk2ISE2aAAIMTUzXmYA')\G ``` ```sql *************************** 1. row *************************** -TIDB_VERSION(): Release Version: v5.1.0-alpha-13-gd5e0ed0aa-dirty -Edition: Community -Git Commit Hash: d5e0ed0aaed72d2f2dfe24e9deec31cb6cb5fdf0 -Git Branch: master -UTC Build Time: 2021-05-24 14:39:20 -GoVersion: go1.13 -Race Enabled: false -TiKV Min Version: v3.0.0-60965b006877ca7234adaced7890d7b029ed1306 -Check Table Before Drop: false -1 row in set (0.00 sec) + tidb_decode_plan('8QIYMAkzMV83CQEH8E85LjA0CWRhdGE6U2VsZWN0aW9uXzYJOTYwCXRpbWU6NzEzLjHCtXMsIGxvb3BzOjIsIGNvcF90YXNrOiB7bnVtOiAxLCBtYXg6IDU2OC41wgErRHByb2Nfa2V5czogMCwgcnBjXxEpAQwFWBAgNTQ5LglZyGNvcHJfY2FjaGVfaGl0X3JhdGlvOiAwLjAwfQkzLjk5IEtCCU4vQQoxCTFfNgkxXz: id task estRows operator info actRows execution info memory disk + TableReader_7 root 319.04 data:Selection_6 960 time:713.1µs, loops:2, cop_task: {num: 1, max: 568.5µs, proc_keys: 0, rpc_num: 1, rpc_time: 549.1µs, copr_cache_hit_ratio: 0.00} 3.99 KB N/A + └─Selection_6 cop[tikv] 319.04 lt(test.t.a, 10000) 960 tikv_task:{time:313.8µs, loops:960} N/A N/A + └─TableFullScan_5 cop[tikv] 960 table:t, keep order:false, stats:pseudo 960 tikv_task:{time:153µs, loops:960} N/A N/A ``` -### TIDB_DECODE_SQL_DIGESTS +## TIDB_DECODE_SQL_DIGESTS The `TIDB_DECODE_SQL_DIGESTS()` function is used to query the normalized SQL statements (a form without formats and arguments) corresponding to the set of SQL digests in the cluster. This function accepts 1 or 2 arguments: @@ -211,8 +302,6 @@ This function returns a string, which is in the format of a JSON string array. T > * This function has a high overhead. In queries with a large number of rows (for example, querying the full table of `information_schema.cluster_tidb_trx` on a large and busy cluster), using this function might cause the queries to run for too long. Use it with caution. > * This function has a high overhead because every time it is called, it internally queries the `STATEMENTS_SUMMARY`, `STATEMENTS_SUMMARY_HISTORY`, `CLUSTER_STATEMENTS_SUMMARY`, and `CLUSTER_STATEMENTS_SUMMARY_HISTORY` tables, and the query involves the `UNION` operation. This function currently does not support vectorization, that is, when calling this function for multiple rows of data, the above query is performed separately for each row. -{{< copyable "sql" >}} - ```sql set @digests = '["e6f07d43b5c21db0fbb9a31feac2dc599787763393dd5acbfad80e247eb02ad5","38b03afa5debbdf0326a014dbe5012a62c51957f1982b3093e748460f8b00821","e5796985ccafe2f71126ed6c0ac939ffa015a8c0744a24b7aee6d587103fd2f7"]'; @@ -250,66 +339,114 @@ See also: - [`Statement Summary Tables`](/statement-summary-tables.md) - [`INFORMATION_SCHEMA.TIDB_TRX`](/information-schema/information-schema-tidb-trx.md) -### TIDB_SHARD +## TIDB_ENCODE_SQL_DIGEST -The `TIDB_SHARD` function can be used to create a shard index to scatter the index hotspot. A shard index is an expression index prefixed with a `TIDB_SHARD` function. +The `TIDB_ENCODE_SQL_DIGEST(query_str)` returns the SQL digest for a query string. -- Creation: +In the following example you can see that both queries get the same query digest, which is because the digest will be `select ?` for both of them. - To create a shard index for the index field `a`, you can use `uk((tidb_shard(a)), a))`. When there is a hotspot caused by monotonically increasing or decreasing data on the index field `a` in the unique secondary index `uk((tidb_shard(a)), a))`, the index's prefix `tidb_shard(a)` can scatter the hotspot to improve the scalability of the cluster. +```sql +SELECT TIDB_ENCODE_SQL_DIGEST('SELECT 1'); +``` -- Scenarios: +``` ++------------------------------------------------------------------+ +| TIDB_ENCODE_SQL_DIGEST('SELECT 1') | ++------------------------------------------------------------------+ +| e1c71d1661ae46e09b7aaec1c390957f0d6260410df4e4bc71b9c8d681021471 | ++------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` - - There is a write hotspot caused by monotonically increasing or decreasing keys on the unique secondary index, and the index contains integer type fields. - - The SQL statement executes an equality query based on all fields of the secondary index, either as a separate `SELECT` or as an internal query generated by `UPDATE`, `DELETE` and so on. The equality query includes two ways: `a = 1` or `a IN (1, 2, ......)`. +```sql +SELECT TIDB_ENCODE_SQL_DIGEST('SELECT 2'); +``` -- Limitations: +``` ++------------------------------------------------------------------+ +| TIDB_ENCODE_SQL_DIGEST('SELECT 2') | ++------------------------------------------------------------------+ +| e1c71d1661ae46e09b7aaec1c390957f0d6260410df4e4bc71b9c8d681021471 | ++------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` - - Cannot be used in inequality queries. - - Cannot be used in queries that contain `OR` mixed with an outmost `AND` operator. - - Cannot be used in the `GROUP BY` clause. - - Cannot be used in the `ORDER BY` clause. - - Cannot be used in the `ON` clause. - - Cannot be used in the `WHERE` subquery. - - Can be used to scatter unique indexes of only the integer fields. - - Might not take effect in composite indexes. - - Cannot go through FastPlan process, which affects optimizer performance. - - Cannot be used to prepare the execution plan cache. +## TIDB_IS_DDL_OWNER -The following example shows how to use the `TIDB_SHARD` function. +The `TIDB_IS_DDL_OWNER()` function returns `1` if the instance you are connected to is the DDL owner. -- Use the `TIDB_SHARD` function to calculate the SHARD value. +```sql +SELECT TIDB_IS_DDL_OWNER(); +``` - The following statement shows how to use the `TIDB_SHARD` function to calculate the SHARD value of `12373743746`: +``` ++---------------------+ +| TIDB_IS_DDL_OWNER() | ++---------------------+ +| 1 | ++---------------------+ +1 row in set (0.00 sec) +``` - {{< copyable "sql" >}} +## TIDB_PARSE_TSO - ```sql - SELECT TIDB_SHARD(12373743746); - ``` +The `TIDB_PARSE_TSO()` function extracts the physical timestamp from a TiDB TSO timestamp. [TSO](/tso.md) stands for Time Stamp Oracle and is a monotonically increasing timestamp given out by PD (Placement Driver) for every transaction. -- The SHARD value is: +A TSO is a number that consists of two parts: - ```sql - +-------------------------+ - | TIDB_SHARD(12373743746) | - +-------------------------+ - | 184 | - +-------------------------+ - 1 row in set (0.00 sec) - ``` +- A physical timestamp +- A logical counter + +```sql +BEGIN; +SELECT TIDB_PARSE_TSO(@@tidb_current_ts); +ROLLBACK; +``` -- Create a shard index using the `TIDB_SHARD` function: +```sql ++-----------------------------------+ +| TIDB_PARSE_TSO(@@tidb_current_ts) | ++-----------------------------------+ +| 2021-05-26 11:33:37.776000 | ++-----------------------------------+ +1 row in set (0.0012 sec) +``` - {{< copyable "sql" >}} +Here `TIDB_PARSE_TSO` is used to extract the physical timestamp from the timestamp number that is available in the `tidb_current_ts` session variable. Because timestamps are given out per transaction, this function is running in a transaction. - ```sql - CREATE TABLE test(id INT PRIMARY KEY CLUSTERED, a INT, b INT, UNIQUE KEY uk((tidb_shard(a)), a)); - ``` +## TIDB_PARSE_TSO_LOGICAL -### TIDB_ROW_CHECKSUM +The `TIDB_PARSE_TSO_LOGICAL(tso)` function returns the logical part of a [TSO](/tso.md) timestamp. -The `TIDB_ROW_CHECKSUM` function is used to query the checksum value of a row. This function can only be used in `SELECT` statements within the FastPlan process. That is, you can query through statements like `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id = ?` or `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id IN (?, ?, ...)`. +```sql +SELECT TIDB_PARSE_TSO_LOGICAL(450456244814610433); +``` + +``` ++--------------------------------------------+ +| TIDB_PARSE_TSO_LOGICAL(450456244814610433) | ++--------------------------------------------+ +| 1 | ++--------------------------------------------+ +1 row in set (0.00 sec) +``` + +```sql +SELECT TIDB_PARSE_TSO_LOGICAL(450456244814610434); +``` + +``` ++--------------------------------------------+ +| TIDB_PARSE_TSO_LOGICAL(450456244814610434) | ++--------------------------------------------+ +| 2 | ++--------------------------------------------+ +1 row in set (0.00 sec) +``` + +## TIDB_ROW_CHECKSUM + +The `TIDB_ROW_CHECKSUM()` function is used to query the checksum value of a row. This function can only be used in `SELECT` statements within the FastPlan process. That is, you can query through statements like `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id = ?` or `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id IN (?, ?, ...)`. To enable the checksum feature of single-row data in TiDB (controlled by the system variable [`tidb_enable_row_level_checksum`](/system-variables.md#tidb_enable_row_level_checksum-new-in-v710)), run the following statement: @@ -342,50 +479,96 @@ The output is as follows: 1 row in set (0.000 sec) ``` -### CURRENT_RESOURCE_GROUP +## TIDB_SHARD -The `CURRENT_RESOURCE_GROUP` function is used to show the resource group name that the current session is bound to. When the [Resource control](/tidb-resource-control.md) feature is enabled, the available resources that can be used by SQL statements are restricted by the resource quota of the bound resource group. +The `TIDB_SHARD()` function creates a shard index to scatter the index hotspot. A shard index is an expression index prefixed with a `TIDB_SHARD()` function. -When a session is established, TiDB binds the session to the resource group that the login user is bound to by default. If the user is not bound to any resource groups, the session is bound to the `default` resource group. Once the session is established, the bound resource group will not change by default, even if the bound resource group of the user is changed via [modifying the resource group bound to the user](/sql-statements/sql-statement-alter-user.md#modify-basic-user-information). To change the bound resource group of the current session, you can use [`SET RESOURCE GROUP`](/sql-statements/sql-statement-set-resource-group.md). +- Creation: -#### Example + To create a shard index for the index field `a`, you can use `uk((tidb_shard(a)), a))`. When there is a hotspot caused by monotonically increasing or decreasing data on the index field `a` in the unique secondary index `uk((tidb_shard(a)), a))`, the index's prefix `tidb_shard(a)` can scatter the hotspot to improve the scalability of the cluster. -Create a user `user1`, create two resource groups `rg1` and `rg2`, and bind the user `user1` to the resource group `rg1`: +- Scenarios: -```sql -CREATE USER 'user1'; -CREATE RESOURCE GROUP 'rg1' RU_PER_SEC = 1000; -CREATE RESOURCE GROUP 'rg2' RU_PER_SEC = 2000; -ALTER USER 'user1' RESOURCE GROUP `rg1`; -``` + - There is a write hotspot caused by monotonically increasing or decreasing keys on the unique secondary index, and the index contains integer type fields. + - The SQL statement executes an equality query based on all fields of the secondary index, either as a separate `SELECT` or as an internal query generated by `UPDATE`, `DELETE` and so on. The equality query includes two ways: `a = 1` or `a IN (1, 2, ......)`. -Use `user1` to log in and view the resource group bound to the current user: +- Limitations: + + - Cannot be used in inequality queries. + - Cannot be used in queries that contain `OR` mixed with an outmost `AND` operator. + - Cannot be used in the `GROUP BY` clause. + - Cannot be used in the `ORDER BY` clause. + - Cannot be used in the `ON` clause. + - Cannot be used in the `WHERE` subquery. + - Can be used to scatter unique indexes of only the integer fields. + - Might not take effect in composite indexes. + - Cannot go through FastPlan process, which affects optimizer performance. + - Cannot be used to prepare the execution plan cache. + +The following example shows how to use the `TIDB_SHARD()` function. + +- Use the `TIDB_SHARD()` function to calculate the SHARD value. + + The following statement shows how to use the `TIDB_SHARD()` function to calculate the SHARD value of `12373743746`: + + ```sql + SELECT TIDB_SHARD(12373743746); + ``` + +- The SHARD value is: + + ```sql + +-------------------------+ + | TIDB_SHARD(12373743746) | + +-------------------------+ + | 184 | + +-------------------------+ + 1 row in set (0.00 sec) + ``` + +- Create a shard index using the `TIDB_SHARD()` function: + + ```sql + CREATE TABLE test(id INT PRIMARY KEY CLUSTERED, a INT, b INT, UNIQUE KEY uk((tidb_shard(a)), a)); + ``` + +## TIDB_VERSION + +The `TIDB_VERSION()` function is used to get the version and build details of the TiDB server that you are connected to. You can use this function when reporting issues on GitHub. ```sql -SELECT CURRENT_RESOURCE_GROUP(); +SELECT TIDB_VERSION()\G ``` -``` -+--------------------------+ -| CURRENT_RESOURCE_GROUP() | -+--------------------------+ -| rg1 | -+--------------------------+ +```sql +*************************** 1. row *************************** +TIDB_VERSION(): Release Version: v{{{ .tidb-version }}} +Edition: Community +Git Commit Hash: d5e0ed0aaed72d2f2dfe24e9deec31cb6cb5fdf0 +Git Branch: master +UTC Build Time: 2021-05-24 14:39:20 +GoVersion: go1.13 +Race Enabled: false +TiKV Min Version: v3.0.0-60965b006877ca7234adaced7890d7b029ed1306 +Check Table Before Drop: false 1 row in set (0.00 sec) ``` -Execute `SET RESOURCE GROUP` to set the resource group for the current session to `rg2`, and then view the resource group bound to the current user: +## VITESS_HASH + +The `VITESS_HASH(num)` function is used to hash a number in the same way Vitess does this. This is to aid migration from Vitess to TiDB. + +Example: ```sql -SET RESOURCE GROUP `rg2`; -SELECT CURRENT_RESOURCE_GROUP(); +SELECT VITESS_HASH(123); ``` ``` -+--------------------------+ -| CURRENT_RESOURCE_GROUP() | -+--------------------------+ -| rg2 | -+--------------------------+ ++---------------------+ +| VITESS_HASH(123) | ++---------------------+ +| 1155070131015363447 | ++---------------------+ 1 row in set (0.00 sec) -``` +``` \ No newline at end of file diff --git a/functions-and-operators/type-conversion-in-expression-evaluation.md b/functions-and-operators/type-conversion-in-expression-evaluation.md index 3ed050120d6ae..6c93c32f63601 100644 --- a/functions-and-operators/type-conversion-in-expression-evaluation.md +++ b/functions-and-operators/type-conversion-in-expression-evaluation.md @@ -1,7 +1,6 @@ --- title: Type Conversion in Expression Evaluation summary: Learn about the type conversion in expression evaluation. -aliases: ['/docs/dev/functions-and-operators/type-conversion-in-expression-evaluation/','/docs/dev/reference/sql/functions-and-operators/type-conversion/'] --- # Type Conversion in Expression Evaluation diff --git a/functions-and-operators/window-functions.md b/functions-and-operators/window-functions.md index 35282ebf4f020..60a357c14a0b1 100644 --- a/functions-and-operators/window-functions.md +++ b/functions-and-operators/window-functions.md @@ -1,7 +1,6 @@ --- title: Window Functions summary: This document introduces window functions supported in TiDB. -aliases: ['/docs/dev/functions-and-operators/window-functions/','/docs/dev/reference/sql/functions-and-operators/window-functions/'] --- # Window Functions diff --git a/garbage-collection-configuration.md b/garbage-collection-configuration.md index 19479c058ce75..440ce613e1ead 100644 --- a/garbage-collection-configuration.md +++ b/garbage-collection-configuration.md @@ -1,7 +1,6 @@ --- title: Garbage Collection Configuration summary: Learn about GC configuration parameters. -aliases: ['/docs/dev/garbage-collection-configuration/','/docs/dev/reference/garbage-collection/configuration/'] --- # Garbage Collection Configuration @@ -23,7 +22,7 @@ For more information about how to modify the value of a system variable, see [Sy > **Note:** > -> This section is only applicable to TiDB Self-Hosted. TiDB Cloud does not have a GC I/O limit by default. +> This section is only applicable to TiDB Self-Managed. TiDB Cloud does not have a GC I/O limit by default. @@ -61,7 +60,7 @@ Based on the `DISTRIBUTED` GC mode, the mechanism of GC in Compaction Filter use > **Note:** > -> The following examples of modifying TiKV configurations are only applicable to TiDB Self-Hosted. For TiDB Cloud, the mechanism of GC in Compaction Filter is enabled by default. +> The following examples of modifying TiKV configurations are only applicable to TiDB Self-Managed. For TiDB Cloud, the mechanism of GC in Compaction Filter is enabled by default. @@ -107,4 +106,16 @@ show config where type = 'tikv' and name like '%enable-compaction-filter%'; | tikv | 172.16.5.36:20163 | gc.enable-compaction-filter | true | | tikv | 172.16.5.35:20163 | gc.enable-compaction-filter | true | +------+-------------------+-----------------------------+-------+ -``` \ No newline at end of file +``` + + + +> **Note:** +> +> When using the Compaction Filter mechanism, GC progress might be delayed, which can affect TiKV scan performance. If your workload contains a large number of coprocessor requests and you observe in the [**TiKV-Details > Coprocessor Detail**](/grafana-tikv-dashboard.md#coprocessor-detail) panel that the `next()` or `prev()` call count in **Total Ops Details** significantly exceeds three times the `processed_keys` calls, you can take the following actions: +> +> - For TiDB versions before v7.1.3, it is recommended to disable Compaction Filter to speed up GC. +> - For TiDB versions from v7.1.3 to v7.5.6, TiDB automatically triggers compaction based on the number of redundant versions in each Region [`region-compact-min-redundant-rows`](/tikv-configuration-file.md#region-compact-min-redundant-rows-new-in-v710) and the percentage of redundant versions [`region-compact-redundant-rows-percent`](/tikv-configuration-file.md#region-compact-redundant-rows-percent-new-in-v710) to improve Compaction Filter GC performance. In this case, adjust these configuration items instead of disabling Compaction Filter. +> - Starting from v7.5.7, [`region-compact-min-redundant-rows`](/tikv-configuration-file.md#region-compact-min-redundant-rows-new-in-v710) and [`region-compact-redundant-rows-percent`](/tikv-configuration-file.md#region-compact-redundant-rows-percent-new-in-v710) are deprecated. TiDB now automatically triggers compaction based on [`gc.auto-compaction.redundant-rows-threshold`](/tikv-configuration-file.md#redundant-rows-threshold-new-in-v757) and [`gc.auto-compaction.redundant-rows-percent-threshold`](/tikv-configuration-file.md#redundant-rows-percent-threshold-new-in-v757). In this case, adjust these configuration items instead of disabling Compaction Filter. + + diff --git a/garbage-collection-overview.md b/garbage-collection-overview.md index b4ac57ac6363b..2c2c0d82f7500 100644 --- a/garbage-collection-overview.md +++ b/garbage-collection-overview.md @@ -1,7 +1,6 @@ --- title: GC Overview summary: Learn about Garbage Collection in TiDB. -aliases: ['/docs/dev/garbage-collection-overview/','/docs/dev/reference/garbage-collection/overview/'] --- # GC Overview diff --git a/generate-self-signed-certificates.md b/generate-self-signed-certificates.md index b286c5ee01d0e..f66809dd01a80 100644 --- a/generate-self-signed-certificates.md +++ b/generate-self-signed-certificates.md @@ -1,7 +1,6 @@ --- title: Generate Self-signed Certificates summary: Use `openssl` to generate self-signed certificates. -aliases: ['/docs/dev/generate-self-signed-certificates/','/docs/dev/how-to/secure/generate-self-signed-certificates/'] --- # Generate Self-Signed Certificates diff --git a/generated-columns.md b/generated-columns.md index 7dac14075c60e..65ee106226dbe 100644 --- a/generated-columns.md +++ b/generated-columns.md @@ -1,7 +1,6 @@ --- title: Generated Columns summary: Learn how to use generated columns. -aliases: ['/docs/dev/generated-columns/','/docs/dev/reference/sql/generated-columns/'] --- # Generated Columns @@ -20,7 +19,7 @@ You can create an index on a generated column whether it is virtual or stored. One of the main usage of generated columns is to extract data from the JSON data type and indexing the data. -In both MySQL 5.7 and TiDB, columns of type JSON cannot be indexed directly. That is, the following table schema is **not supported**: +In both MySQL 8.0 and TiDB, columns of type JSON cannot be indexed directly. That is, the following table schema is **not supported**: {{< copyable "sql" >}} @@ -153,5 +152,6 @@ The current limitations of JSON and generated columns are as follows: - You cannot add a stored generated column through `ALTER TABLE`. - You can neither convert a stored generated column to a normal column through the `ALTER TABLE` statement nor convert a normal column to a stored generated column. - You cannot modify the expression of a stored generated column through the `ALTER TABLE` statement. -- Not all [JSON functions](/functions-and-operators/json-functions.md) are supported; +- Not all [JSON functions](/functions-and-operators/json-functions.md) are supported. +- The [`NULLIF()` function](/functions-and-operators/control-flow-functions.md) is not supported. You can use the [`CASE` function](/functions-and-operators/control-flow-functions.md) instead. - Currently, the generated column index replacement rule is valid only when the generated column is a virtual generated column. It is not valid on the stored generated column, but the index can still be used by directly using the generated column itself. diff --git a/geo-distributed-deployment-topology.md b/geo-distributed-deployment-topology.md index 230dff217355e..c5a149c99a797 100644 --- a/geo-distributed-deployment-topology.md +++ b/geo-distributed-deployment-topology.md @@ -1,7 +1,6 @@ --- title: Geo-distributed Deployment topology summary: Learn the geo-distributed deployment topology of TiDB. -aliases: ['/docs/dev/geo-distributed-deployment-topology/'] --- # Geo-Distributed Deployment Topology @@ -17,6 +16,10 @@ This document takes the typical architecture of three data centers (DC) in two c | TiKV | 5 | 16 VCore 32GB 4TB (nvme ssd) * 1 | 10.0.1.11
10.0.1.12
10.0.1.13
10.0.1.14 | 10.0.1.15 | Default port
Global directory configuration | | Monitoring & Grafana | 1 | 4 VCore 8GB * 1 500GB (ssd) | 10.0.1.16 | | Default port
Global directory configuration | +> **Note:** +> +> The IP addresses of the instances are given as examples only. In your actual deployment, replace the IP addresses with your actual IP addresses. + ### Topology templates - [The geo-distributed topology template](https://github.com/pingcap/docs/blob/master/config-templates/geo-redundancy-deployment.yaml) @@ -57,10 +60,14 @@ This section describes the key parameter configuration of the TiDB geo-distribut - To prevent remote TiKV nodes from launching unnecessary Raft elections, it is required to increase the minimum and maximum number of ticks that the remote TiKV nodes need to launch an election. The two parameters are set to `0` by default. ```yaml - raftstore.raft-min-election-timeout-ticks: 1000 - raftstore.raft-max-election-timeout-ticks: 1020 + raftstore.raft-min-election-timeout-ticks: 50 + raftstore.raft-max-election-timeout-ticks: 60 ``` +> **Note:** +> +> Using `raftstore.raft-min-election-timeout-ticks` and `raftstore.raft-max-election-timeout-ticks` to configure larger election timeout ticks for a TiKV node can significantly decrease the likelihood of Regions on that node becoming Leaders. However, in a disaster scenario where some TiKV nodes are offline and the remaining active TiKV nodes lag behind in Raft logs, only Regions on this TiKV node with large election timeout ticks can become Leaders. Because Regions on this TiKV node must wait for at least the duration set by `raftstore.raft-min-election-timeout-ticks' before initiating an election, it is recommended to avoid setting these values excessively large to prevent potential impact on the cluster availability in such scenarios. + #### PD parameters - The PD metadata information records the topology of the TiKV cluster. PD schedules the Raft Group replicas on the following four dimensions: diff --git a/get-started-with-tidb-lightning.md b/get-started-with-tidb-lightning.md index d2b406961a544..d48ebf7813db7 100644 --- a/get-started-with-tidb-lightning.md +++ b/get-started-with-tidb-lightning.md @@ -1,55 +1,68 @@ --- -title: Get Started with TiDB Lightning -summary: Learn how to deploy TiDB Lightning and import full backup data to TiDB. -aliases: ['/docs/dev/get-started-with-tidb-lightning/','/docs/dev/how-to/get-started/tidb-lightning/'] +title: Quick Start for TiDB Lightning +summary: TiDB Lightning is a tool for importing MySQL data into a TiDB cluster. It is recommended for test and trial purposes only, not for production or development environments. The process involves preparing full backup data, deploying the TiDB cluster, installing TiDB Lightning, starting TiDB Lightning, and checking data integrity. For detailed features and usage, refer to the TiDB Lightning Overview. --- -# Get Started with TiDB Lightning +# Quick Start for TiDB Lightning -This tutorial assumes you use several new and clean CentOS 7 instances. You can use VMware, VirtualBox or other tools to deploy a virtual machine locally or a small cloud virtual machine on a vendor-supplied platform. Because TiDB Lightning consumes a large amount of computer resources, it is recommended that you allocate at least 16 GB memory and CPU of 32 cores for running it with the best performance. +This document provides a quick guide on getting started with TiDB Lightning by importing MySQL data into a TiDB cluster. > **Warning:** > > The deployment method in this tutorial is only recommended for test and trial. **Do not apply it in the production or development environment.** -## Prepare full backup data +## Step 1: Prepare full backup data -First, use [`dumpling`](/dumpling-overview.md) to export data from MySQL: +First, you can use [dumpling](/dumpling-overview.md) to export data from MySQL. -{{< copyable "shell-regular" >}} +1. Run `tiup --version` to check if TiUP is already installed. If TiUP is installed, skip this step. If TiUP is not installed, run the following command: -```sh -tiup dumpling -h 127.0.0.1 -P 3306 -u root -t 16 -F 256MB -B test -f 'test.t[12]' -o /data/my_database/ -``` + ``` + curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh + ``` + +2. Using TiUP to install Dumpling: + + ```shell + tiup install dumpling + ``` + +3. To export data from MySQL, you can refer to the detailed steps provided in [Use Dumpling to Export Data](/dumpling-overview.md#export-to-sql-files): + + ```sh + tiup dumpling -h 127.0.0.1 -P 3306 -u root -t 16 -F 256MB -B test -f 'test.t[12]' -o /data/my_database/ + ``` + + In the above command: + + - `-t 16`: Export data using 16 threads. + - `-F 256MB`: Split each table into multiple files, with each file approximately 256 MB in size. + - `-B test`: Export from the `test` database. + - `-f 'test.t[12]'`: Export only the two tables `test.t1` and `test.t2`. -In the above command: + The full backup data exported will be saved in the `/data/my_database` directory. -- `-B test`: means the data is exported from the `test` database. -- `-f test.t[12]`: means only the `test.t1` and `test.t2` tables are exported. -- `-t 16`: means 16 threads are used to export the data. -- `-F 256MB`: means a table is partitioned into chunks and one chunk is 256 MB. +## Step 2: Deploy the TiDB cluster -After executing this command, the full backup data is exported to the `/data/my_database` directory. +Before starting the data import, you need to deploy a TiDB cluster for the import. If you already have a TiDB cluster, you can skip this step. -## Deploy TiDB Lightning +For the steps on deploying a TiDB cluster, refer to the [Quick Start Guide for the TiDB Database Platform](/quick-start-with-tidb.md). -### Step 1: Deploy a TiDB cluster +## Step 3: Install TiDB Lightning -Before the data import, you need to deploy a TiDB cluster. In this tutorial, TiDB v5.4.0 is used as an example. For the deployment method, refer to [Deploy a TiDB Cluster Using TiUP](/production-deployment-using-tiup.md). +Run the following command to install the latest version of TiDB Lightning: -### Step 2: Download TiDB Lightning installation package +```shell +tiup install tidb-lightning +``` -The TiDB Lightning installation package is included in the TiDB Toolkit. To download the TiDB Toolkit, see [Download TiDB Tools](/download-ecosystem-tools.md). +## Step 4: Start TiDB Lightning > **Note:** > -> TiDB Lightning is compatible with TiDB clusters of earlier versions. It is recommended that you download the latest stable version of the TiDB Lightning installation package. - -### Step 3: Start `tidb-lightning` +> The import method in this section is only suitable for testing and functional experience. For production environments, refer to [Migrate Large Datasets from MySQL to TiDB](/migrate-large-mysql-to-tidb.md) -1. Upload `bin/tidb-lightning` and `bin/tidb-lightning-ctl` in the package to the server where TiDB Lightning is deployed. -2. Upload the [prepared data source](#prepare-full-backup-data) to the server. -3. Configure `tidb-lightning.toml` as follows: +1. Create the configuration file `tidb-lightning.toml` and fill in the following settings based on your cluster information: ```toml [lightning] @@ -60,8 +73,7 @@ The TiDB Lightning installation package is included in the TiDB Toolkit. To down [tikv-importer] # Configure the import mode backend = "local" - # Sets the directory for temporarily storing the sorted key-value pairs. - # The target directory must be empty. + # Sets the directory for temporarily storing the sorted key-value pairs. The target directory must be empty. sorted-kv-dir = "/mnt/ssd/sorted-kv-dir" [mydumper] @@ -83,16 +95,14 @@ The TiDB Lightning installation package is included in the TiDB Toolkit. To down pd-addr = "172.16.31.3:2379" ``` -4. After configuring the parameters properly, use a `nohup` command to start the `tidb-lightning` process. If you directly run the command in the command-line, the process might exit because of the SIGHUP signal received. Instead, it's preferable to run a bash script that contains the `nohup` command: +2. Run `tidb-lightning`. To avoid the program exiting due to the `SIGHUP` signal when starting the program directly in the command line using `nohup`, it is recommended to put the `nohup` command in a script. For example: - {{< copyable "shell-regular" >}} - - ```sh + ```shell #!/bin/bash nohup tiup tidb-lightning -config tidb-lightning.toml > nohup.out & ``` -### Step 4: Check data integrity +## Step 5: Check data integrity After the import is completed, TiDB Lightning exits automatically. If the import is successful, you can find `tidb lightning exit` in the last line of the log file. diff --git a/glossary.md b/glossary.md index 2f5b5ee5f6b98..ae979db4ab95c 100644 --- a/glossary.md +++ b/glossary.md @@ -1,7 +1,6 @@ --- title: Glossary summary: Glossaries about TiDB. -aliases: ['/docs/dev/glossary/'] --- # Glossary @@ -76,6 +75,10 @@ Leader/Follower/Learner each corresponds to a role in a Raft group of [peers](#r Starting from v5.0, TiDB introduces Massively Parallel Processing (MPP) architecture through TiFlash nodes, which shares the execution workloads of large join queries among TiFlash nodes. When the MPP mode is enabled, TiDB, based on cost, determines whether to use the MPP framework to perform the calculation. In the MPP mode, the join keys are redistributed through the Exchange operation while being calculated, which distributes the calculation pressure to each TiFlash node and speeds up the calculation. For more information, see [Use TiFlash MPP Mode](/tiflash/use-tiflash-mpp-mode.md). +### Multi-version concurrency control (MVCC) + +[MVCC](https://en.wikipedia.org/wiki/Multiversion_concurrency_control) is a concurrency control mechanism in TiDB and other databases. It processes the memory read by transactions to achieve concurrent access to TiDB, thereby avoiding blocking caused by conflicts between concurrent reads and writes. + ## O ### Old value @@ -169,7 +172,3 @@ Top SQL helps locate SQL queries that contribute to a high load of a TiDB or TiK ### TSO Because TiKV is a distributed storage system, it requires a global timing service, Timestamp Oracle (TSO), to assign a monotonically increasing timestamp. In TiKV, such a feature is provided by PD, and in Google [Spanner](http://static.googleusercontent.com/media/research.google.com/en//archive/spanner-osdi2012.pdf), this feature is provided by multiple atomic clocks and GPS. - -### TTL - -[Time to live (TTL)](/time-to-live.md) is a feature that allows you to manage TiDB data lifetime at the row level. For a table with the TTL attribute, TiDB automatically checks data lifetime and deletes expired data at the row level. diff --git a/grafana-overview-dashboard.md b/grafana-overview-dashboard.md index daf0ba5c74976..eef5ff710d8c3 100644 --- a/grafana-overview-dashboard.md +++ b/grafana-overview-dashboard.md @@ -1,7 +1,6 @@ --- title: Key Metrics summary: Learn some key metrics displayed on the Grafana Overview dashboard. -aliases: ['/docs/dev/grafana-overview-dashboard/','/docs/dev/reference/key-monitoring-metrics/overview-dashboard/'] --- # Key Metrics @@ -42,7 +41,7 @@ To understand the key metrics displayed on the Overview dashboard, check the fol | TiDB | Transaction Duration | The execution time of a transaction | | TiDB | KV Cmd OPS | The number of executed KV commands. | | TiDB | KV Cmd Duration 99 | The execution time of the KV command. | -| TiDB | PD TSO OPS | The number of TSO that TiDB obtains from PD per second. | +| TiDB | PD TSO OPS | The number of gRPC requests per second that TiDB sends to PD (cmd) and the number of TSO requests (request); each gRPC request contains a batch of TSO requests. | | TiDB | PD TSO Wait Duration | The duration that TiDB waits for PD to return TSO. | | TiDB | TiClient Region Error OPS | The number of Region related errors returned by TiKV. | | TiDB | Lock Resolve OPS | The number of TiDB operations that resolve locks. When TiDB's read or write request encounters a lock, it tries to resolve the lock. | diff --git a/grafana-pd-dashboard.md b/grafana-pd-dashboard.md index 13790d5aa6f41..c1e6029a3f466 100644 --- a/grafana-pd-dashboard.md +++ b/grafana-pd-dashboard.md @@ -1,7 +1,6 @@ --- title: Key Monitoring Metrics of PD summary: Learn some key metrics displayed on the Grafana PD dashboard. -aliases: ['/docs/dev/grafana-pd-dashboard/','/docs/dev/reference/key-monitoring-metrics/pd-dashboard/'] --- # Key Monitoring Metrics of PD @@ -78,7 +77,7 @@ The following is the description of PD Dashboard metrics items: - Store Write rate keys: The total written keys on each TiKV instance - Hot cache write entry number: The number of peers on each TiKV instance that are in the write hotspot statistics module - Selector events: The event count of Selector in the hotspot scheduling module -- Direction of hotspot move leader: The direction of leader movement in the hotspot scheduling. The positive number means scheduling into the instance. The negtive number means scheduling out of the instance +- Direction of hotspot move leader: The direction of leader movement in the hotspot scheduling. The positive number means scheduling into the instance. The negative number means scheduling out of the instance - Direction of hotspot move peer: The direction of peer movement in the hotspot scheduling. The positive number means scheduling into the instance. The negative number means scheduling out of the instance ![PD Dashboard - Hot write metrics](/media/pd-dashboard-hotwrite-v4.png) diff --git a/grafana-performance-overview-dashboard.md b/grafana-performance-overview-dashboard.md index f2e6c1005e5e6..3aa9ae07d9348 100644 --- a/grafana-performance-overview-dashboard.md +++ b/grafana-performance-overview-dashboard.md @@ -67,11 +67,11 @@ Number of commands processed by all TiDB instances per second based on type ### KV/TSO Request OPS - kv request total: Total number of KV requests per second in all TiDB instances -- kv request by type: Number of KV requests per second in all TiDB instances based on such types as `Get`, `Prewrite`, and `Commit`. -- tso - cmd: Number of `tso cmd` requests per second in all TiDB instances -- tso - request: Number of `tso request` requests per second in all TiDB instances +- kv request by type: Number of KV requests per second in all TiDB instances based on such types as `Get`, `Prewrite`, and `Commit` +- tso - cmd: Number of gRPC requests per second that TiDB sends to PD in all TiDB instances; each gRPC request contains a batch of TSO requests +- tso - request: Number of TSO requests per second in all TiDB instances -Generally, dividing `tso - cmd` by `tso - request` yields the average batch size of requests per second. +Generally, `tso - request` divided by `tso - cmd` is the average size of TSO request batches per second. ### KV Request Time By Source @@ -138,10 +138,10 @@ Average time consumed in executing gRPC requests in all TiKV instances based on ### PD TSO Wait/RPC Duration -- wait - avg: Average time in waiting for PD to return TSO in all TiDB instances -- rpc - avg: Average time from sending TSO requests to PD to receiving TSO in all TiDB instances -- wait - 99: P99 time in waiting for PD to return TSO in all TiDB instances -- rpc - 99: P99 time from sending TSO requests to PD to receiving TSO in all TiDB instances +- wait - avg: Average duration of waiting for PD to return TSO in all TiDB instances +- rpc - avg: Average duration from the time that TiDB sends gRPC requests to PD to get TSO to the time that TiDB receives TSO in all TiDB instances +- wait - 99: P99 duration of waiting for PD to return TSO in all TiDB instances +- rpc - 99: P99 duration from the time that TiDB sends gRPC requests to PD to get TSO to the time that TiDB receives TSO in all TiDB instances ### Storage Async Write Duration, Store Duration, and Apply Duration diff --git a/grafana-resource-control-dashboard.md b/grafana-resource-control-dashboard.md index 3cca223262ba3..0c93e147fb7a1 100644 --- a/grafana-resource-control-dashboard.md +++ b/grafana-resource-control-dashboard.md @@ -11,7 +11,7 @@ The Grafana dashboard is divided into a series of sub dashboards which include O If your cluster has used the [Resource Control](/tidb-resource-control.md) feature, you can get an overview of the resource consumption status from the Resource Control dashboard. -TiDB uses the [token bucket algorithm](https://en.wikipedia.org/wiki/Token_bucket) for flow control. As described in the [RFC: Global Resource Control in TiDB](https://github.com/pingcap/tidb/blob/master/docs/design/2022-11-25-global-resource-control.md#distributed-token-buckets), a TiDB node might have multiple Resource Groups, which are flow controlled by GAC (Global Admission Control) on the PD side. The Local Token Buckets in each TiDB node periodically (5 seconds by default) communicate with the GAC on the PD side to reconfigure the local tokens. In TiDB, the Local Token Buckets are implemented as Resource Controller Clients. +TiDB uses the [token bucket algorithm](https://en.wikipedia.org/wiki/Token_bucket) for flow control. As described in the [RFC: Global Resource Control in TiDB](https://github.com/pingcap/tidb/blob/release-7.5/docs/design/2022-11-25-global-resource-control.md#distributed-token-buckets), a TiDB node might have multiple Resource Groups, which are flow controlled by GAC (Global Admission Control) on the PD side. The Local Token Buckets in each TiDB node periodically (5 seconds by default) communicate with the GAC on the PD side to reconfigure the local tokens. In TiDB, the Local Token Buckets are implemented as Resource Controller Clients. This document describes some key monitoring metrics displayed on the Resource Control dashboard. diff --git a/grafana-tidb-dashboard.md b/grafana-tidb-dashboard.md index 3e6427576186f..08fca1da11335 100644 --- a/grafana-tidb-dashboard.md +++ b/grafana-tidb-dashboard.md @@ -1,7 +1,6 @@ --- title: TiDB Monitoring Metrics summary: Learn some key metrics displayed on the Grafana TiDB dashboard. -aliases: ['/docs/dev/grafana-tidb-dashboard/','/docs/dev/reference/key-monitoring-metrics/tidb-dashboard/'] --- # TiDB Monitoring Metrics @@ -133,10 +132,10 @@ The following metrics relate to requests sent to TiKV. Retry requests are counte - PD Client CMD OPS: the statistics of commands executed by PD Client per second - PD Client CMD Duration: the time it takes for PD Client to execute commands - PD Client CMD Fail OPS: the statistics of failed commands executed by PD Client per second -- PD TSO OPS: the number of TSO that TiDB obtains from PD per second +- PD TSO OPS: the number of gRPC requests per second that TiDB sends to PD (cmd) and the number of TSO requests (request); each gRPC request contains a batch of TSO requests - PD TSO Wait Duration: the time that TiDB waits for PD to return TSO -- PD TSO RPC duration: the duration from the time that TiDB sends request to PD (to get TSO) to the time that TiDB receives TSO -- Start TSO Wait Duration: the duration from the time that TiDB sends request to PD (to get `start TSO`) to the time that TiDB receives `start TSO` +- PD TSO RPC duration: the duration from the time that TiDB sends gRPC requests to PD to get TSO to the time that TiDB receives the gRPC response from PD +- Async TSO Duration: the duration from the time that TiDB prepares to get TSO to the time that TiDB actually starts to wait for PD to return TSO ### Schema Load diff --git a/grafana-tikv-dashboard.md b/grafana-tikv-dashboard.md index a50f74c644fff..a9bfac112787c 100644 --- a/grafana-tikv-dashboard.md +++ b/grafana-tikv-dashboard.md @@ -1,7 +1,6 @@ --- title: Key Monitoring Metrics of TiKV summary: Learn some key metrics displayed on the Grafana TiKV dashboard. -aliases: ['/docs/dev/grafana-tikv-dashboard/','/docs/dev/reference/key-monitoring-metrics/tikv-dashboard/'] --- # Key Monitoring Metrics of TiKV @@ -185,6 +184,21 @@ This section provides a detailed description of these key metrics on the **TiKV- ![TiKV Dashboard - Storage metrics](/media/tikv-dashboard-storage.png) +### Flow Control + +- Scheduler flow: The scheduler traffic on each TiKV instance in real time. +- Scheduler discard ratio: The rejection ratio of scheduler requests on each TiKV instance. If this ratio is greater than 0, it indicates that flow control exists. When `Compaction pending bytes` exceeds its threshold, TiKV will linearly increase the `Scheduler discard ratio` based on the exceeded portion. The client will retry the rejected requests automatically. +- Throttle duration: The blocked duration for the execution of the scheduler requests when flow control is triggered due to too many L0 files. If this metric has values, it indicates that flow control exists. +- Scheduler throttled CF: The CF that triggers RocksDB throttling when the flow control threshold is reached. +- Flow controller actions: The actions that trigger RocksDB throttling when the flow control threshold is reached. +- Flush/L0 flow: The traffic of flush and L0 compaction for different CFs of RocksDB on each TiKV instance. +- Flow control factors: The factors related to triggering RocksDB throttling. +- Compaction pending bytes: The size of the RocksDB data awaiting compaction in real time on each TiKV instance. +- Txn command throttled duration: The blocked duration for commands related to transactions due to throttling. Under normal circumstances, this metric is 0. +- Non-txn command throttled duration: The blocked duration for other commands due to throttling. Under normal circumstances, this metric is 0. + +![TiKV Dashboard - Flow Control metrics](/media/tikv-dashboard-flow-control.png) + ### Scheduler - Scheduler stage total: The number of commands at each stage per second. There should not be a lot of errors in a short time. @@ -444,6 +458,79 @@ This section provides a detailed description of these key metrics on the **TiKV- - Encrypt/decrypt data nanos: The histogram of duration on encrypting/decrypting data each time - Read/write encryption meta duration: The time consumed for reading/writing encryption meta files +### Log Backup + +- Handle Event Rate: The speed of handling write events +- Initial Scan Generate Event Throughput: Incremental scanning speed when generating a new listener stream +- Abnormal Checkpoint TS Lag: The lag of the current checkpoint TS to the present time for each task +- Memory Of Events: An estimated amount of memory occupied by temporary data generated by incremental scanning +- Observed Region Count: The number of Regions currently listened to +- Errors: The number and type of retryable and non-fatal errors +- Fatal Errors: The number and type of fatal errors. Usually, fatal errors cause the task to be paused. +- Checkpoint TS of Tasks: Checkpoint TS for each task +- Flush Duration: The heat map of how long it takes for moving cached data to external storage +- Initial Scanning Duration: The heat map of how long it takes for incremental scanning when creating a new listening stream +- Convert Raft Event Duration: The heat map of how long it takes to transform a Raft log entry into backup data after creating a listening stream +- Command Batch Size: The batch size (within a single Raft group) of the listening Raft command +- Save to Temp File Duration: The heat map of how long it takes to temporarily store a batch of backup data (spanning several tasks) into the temporary file area +- Write to Temp File Duration: The heat map of how long it takes to temporarily store a batch of backup data (from a particular task) into the temporary file area +- System Write Call Duration: The heat map of how long it takes to write a batch of backup data (from a Region) to a temporary file +- Internal Message Type: The type of messages received by the actor responsible for the log backup within TiKV +- Internal Message Handling Duration (P90|P99): The speed of consuming and processing each type of messages +- Initial Scan RocksDB Throughput: The read traffic generated by RocksDB internal logging during incremental scanning +- Initial Scan RocksDB Operation: The number of individual operations logged internally by RocksDB during incremental scanning +- Initial Scanning Trigger Reason: The reason for triggering incremental scanning +- Region Checkpoint Key Putting: The number of checkpoint operations logged to the PD + +> **Note:** +> +> The following monitoring metrics all use TiDB nodes as their data source, but they have some impact on the log backup process. Therefore, they are placed in the **TiKV Details** dashboard for ease of reference. TiKV actively pushes progress most of the time, but it is normal for some of the following monitoring metrics to occasionally not have sampled data. + +- Request Checkpoint Batch Size: The request batch size when the log backup coordinator requests checkpoint information for each TiKV +- Tick Duration \[P99|P90\]: The time taken by the tick inside the coordinator +- Region Checkpoint Failure Reason: The reason why a Region checkpoint cannot advance within the coordinator +- Request Result: The record of the coordinator's success or failure in advancing the Region checkpoint +- Get Region Operation Count: The number of times the coordinator requests Region information from the PD +- Try Advance Trigger Time: The time taken for the coordinator to attempt to advance the checkpoint + +### Backup & Import + +- Import CPU Utilization: The CPU utilization aggregated by SST importer. +- Import Thread Count: The number of threads used by SST importer. +- Import Errors: The number of errors encountered during SST import. +- Import RPC Duration: The time spent on various RPC calls in SST importer. +- Import RPC Ops: The total number of RPC calls in SST importer. +- Import RPC Count: The number of RPC calls being processed by SST importer. +- Import Write/Download RPC Duration: The RPC time for write or download operations in SST importer. +- Import Wait Duration: The time spent waiting in queue for download task execution. +- Import Read SST Duration: The time spent reading an SST file from external storage and downloading it to TiKV. +- Import Rewrite SST Duration: The time spent rewriting the SST file based on rewrite rules. +- Import Ingest RPC Duration: The time spent handling ingest RPC requests on TiKV. +- Import Ingest SST Duration: The time spent ingesting the SST file into RocksDB. +- Import Ingest SST Bytes: The number of bytes ingested. +- Import Download SST Throughput: The SST download throughput in bytes per second. +- cloud request: The number of requests to cloud providers. + +### Point In Time Restore + +- CPU Usage: The CPU utilization by point-in-time recovery (PITR). +- P99 RPC Duration: The 99th percentile of RPC request duration. +- Import RPC Ops: The total number of RPC calls in SST importer. +- Import RPC Count: The number of RPC calls being processed by SST importer. +- Cache Events: The number of events in the file cache during SST import. +- Overall RPC Duration: The time spent on RPC calls. +- Read File into Memory Duration: The time spent downloading files from external storage and loading them into memory. +- Queuing Time: The time spent waiting to be scheduled on a thread. +- Apply Request Throughput: The rate of applying requests in bytes. +- Downloaded File Size: The size of downloaded file in bytes. +- Apply Batch Size: The number of bytes for applying to Raft store in one batch. +- Blocked by Concurrency Time: The time spent waiting for execution due to concurrency constraints. +- Apply Request Speed: The speed of applying request to Raft store. +- Cached File in Memory: The files cached by the applying requests of SST importer. +- Engine Requests Unfinished: The number of pending requests to Raft store. +- Apply Time: The time spent writing data to Raft store. +- Raft Store Memory Usage: The memory usage for Raft store. + ### Explanation of Common Parameters #### gRPC Message Type diff --git a/hardware-and-software-requirements.md b/hardware-and-software-requirements.md index 03934b25a4428..4058ce13412b9 100644 --- a/hardware-and-software-requirements.md +++ b/hardware-and-software-requirements.md @@ -1,7 +1,6 @@ --- title: Software and Hardware Recommendations summary: Learn the software and hardware recommendations for deploying and running TiDB. -aliases: ['/docs/dev/hardware-and-software-requirements/','/docs/dev/how-to/deploy/hardware-recommendations/'] --- # Software and Hardware Recommendations @@ -18,84 +17,107 @@ As an open-source distributed SQL database with high performance, TiDB can be de ## OS and platform requirements - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Operating systemsSupported CPU architectures
Red Hat Enterprise Linux 8.4 or a later 8.x version
  • x86_64
  • ARM 64
  • Red Hat Enterprise Linux 7.3 or a later 7.x version
  • CentOS 7.3 or a later 7.x version
  • x86_64
  • ARM 64
Amazon Linux 2
  • x86_64
  • ARM 64
Kylin Euler V10 SP1/SP2
  • x86_64
  • ARM 64
UnionTech OS (UOS) V20
  • x86_64
  • ARM 64
openEuler 22.03 LTS SP1
  • x86_64
  • ARM 64
macOS 12 (Monterey) or later
  • x86_64
  • ARM 64
Oracle Enterprise Linux 7.3 or a later 7.x versionx86_64
Ubuntu LTS 18.04 or laterx86_64
CentOS 8 Stream
  • x86_64
  • ARM 64
Debian 9 (Stretch) or laterx86_64
Fedora 35 or laterx86_64
openSUSE Leap later than v15.3 (not including Tumbleweed)x86_64
Rocky Linux 9.1 or later
  • x86_64
  • ARM 64
SUSE Linux Enterprise Server 15x86_64
- -> **Note:** -> -> - For Oracle Enterprise Linux, TiDB supports the Red Hat Compatible Kernel (RHCK) and does not support the Unbreakable Enterprise Kernel provided by Oracle Enterprise Linux. -> - According to [CentOS Linux EOL](https://www.centos.org/centos-linux-eol/), the upstream support for CentOS Linux 8 ended on December 31, 2021. CentOS Stream 8 continues to be supported by the CentOS organization. -> - Support for Ubuntu 16.04 will be removed in future versions of TiDB. Upgrading to Ubuntu 18.04 or later is strongly recommended. -> - If you are using the 32-bit version of an operating system listed in the preceding table, TiDB **is not guaranteed** to be compilable, buildable or deployable on the 32-bit operating system and the corresponding CPU architecture, or TiDB does not actively adapt to the 32-bit operating system. -> - Other operating system versions not mentioned above might work but are not officially supported. +In v7.5 LTS, TiDB ensures multi-level quality standards for various combinations of operating systems and CPU architectures. + ++ For the following combinations of operating systems and CPU architectures, TiDB **provides enterprise-level production quality**, and the product features have been comprehensively and systematically verified: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Operating systemsSupported CPU architectures
Red Hat Enterprise Linux 8.4 or a later 8.x version
  • x86_64
  • ARM 64
  • Red Hat Enterprise Linux 7.3 or a later 7.x version
  • CentOS 7.3 or a later 7.x version (TiDB ends support for it starting from 8.4 DMR)
  • x86_64
  • ARM 64
Amazon Linux 2
  • x86_64
  • ARM 64
Rocky Linux 9.1 or later
  • x86_64
  • ARM 64
Kylin V10 SP1/SP2/SP3 (SP3 is supported starting from v7.5.5)
  • x86_64
  • ARM 64
UnionTech OS (UOS) V20
  • x86_64
  • ARM 64
openEuler 22.03 LTS SP1/SP3 (before v7.5.6), SP3/SP4 (supported starting from v7.5.6)
  • x86_64
  • ARM 64
+ + > **Note:** + > + > According to [CentOS Linux EOL](https://blog.centos.org/2023/04/end-dates-are-coming-for-centos-stream-8-and-centos-linux-7/), the upstream support for CentOS Linux 7 ends on June 30, 2024. TiDB ends the support for CentOS 7 starting from the 8.4 DMR version. It is recommended to use Rocky Linux 9.1 or a later version. + ++ For the following combinations of operating systems and CPU architectures, you can compile, build, and deploy TiDB. In addition, you can also use the basic features of OLTP, OLAP, and the data tools. However, TiDB **does not guarantee enterprise-level production quality**: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Operating systemsSupported CPU architectures
macOS 12 (Monterey) or later
  • x86_64
  • ARM 64
Oracle Enterprise Linux 7.3 or a later 7.x versionx86_64
Ubuntu LTS 18.04 or laterx86_64
CentOS Stream 8
  • x86_64
  • ARM 64
Debian 9 (Stretch) or laterx86_64
Fedora 35 or laterx86_64
openSUSE Leap later than v15.3 (excluding Tumbleweed)x86_64
SUSE Linux Enterprise Server 15x86_64
+ + > **Note:** + > + > - For Oracle Enterprise Linux, TiDB supports the Red Hat Compatible Kernel (RHCK), but does not support the Unbreakable Enterprise Kernel provided by Oracle Enterprise Linux. + > - Support for Ubuntu 16.04 will be removed in future versions of TiDB. Upgrading to Ubuntu 18.04 or later is strongly recommended. + > - CentOS Stream 8 reaches [End of Builds](https://blog.centos.org/2023/04/end-dates-are-coming-for-centos-stream-8-and-centos-linux-7/) on May 31, 2024. + ++ If you are using the 32-bit version of an operating system listed in the preceding two tables, TiDB **is not guaranteed** to be compilable, buildable or deployable on the 32-bit operating system and the corresponding CPU architecture, or TiDB does not actively adapt to the 32-bit operating system. + ++ Other operating system versions not mentioned above might work but are not officially supported. ### Libraries required for compiling and running TiDB @@ -112,7 +134,7 @@ Library required for running TiDB: glibc (2.28-151.el8 version) The following CPU architectures are supported: -- x86_64. Starting from TiDB v6.6.0, the [x84-64-v2 instruction set](https://developers.redhat.com/blog/2021/01/05/building-red-hat-enterprise-linux-9-for-the-x86-64-v2-microarchitecture-level) is required. +- x86_64. Starting from TiDB v6.6.0, the [x86-64-v2 instruction set](https://developers.redhat.com/blog/2021/01/05/building-red-hat-enterprise-linux-9-for-the-x86-64-v2-microarchitecture-level) is required. - ARM 64 ## Software recommendations @@ -144,7 +166,7 @@ You can deploy and run TiDB on the 64-bit generic hardware server platform in th | Component | CPU | Memory | Local Storage | Network | Number of Instances (Minimum Requirement) | | :------: | :-----: | :-----: | :----------: | :------: | :----------------: | -| TiDB | 8 core+ | 16 GB+ | No special requirements | Gigabit network card | 1 (can be deployed on the same machine with PD) | +| TiDB | 8 core+ | 16 GB+ | [Storage requirements](#storage-requirements) | Gigabit network card | 1 (can be deployed on the same machine with PD) | | PD | 4 core+ | 8 GB+ | SAS, 200 GB+ | Gigabit network card | 1 (can be deployed on the same machine with TiDB) | | TiKV | 8 core+ | 32 GB+ | SAS, 200 GB+ | Gigabit network card | 3 | | TiFlash | 32 core+ | 64 GB+ | SSD, 200 GB+ | Gigabit network card | 1 | @@ -156,7 +178,6 @@ You can deploy and run TiDB on the 64-bit generic hardware server platform in th > - For performance-related test, do not use low-performance storage and network hardware configuration, in order to guarantee the correctness of the test result. > - For the TiKV server, it is recommended to use NVMe SSDs to ensure faster reads and writes. > - If you only want to test and verify the features, follow [Quick Start Guide for TiDB](/quick-start-with-tidb.md) to deploy TiDB on a single machine. -> - The TiDB server uses the disk to store server logs, so there are no special requirements for the disk type and capacity in the test environment. > - Starting from v6.3.0, to deploy TiFlash under the Linux AMD64 architecture, the CPU must support the AVX2 instruction set. Ensure that `cat /proc/cpuinfo | grep avx2` has output. To deploy TiFlash under the Linux ARM64 architecture, the CPU must support the ARMv8 instruction set architecture. Ensure that `cat /proc/cpuinfo | grep 'crc32' | grep 'asimd'` has output. By using the instruction set extensions, TiFlash's vectorization engine can deliver better performance. ### Production environment @@ -221,7 +242,7 @@ As an open-source distributed SQL database, TiDB requires the following network | Alertmanager | 9093 | the port for the alert web service | | Alertmanager | 9094 | the alert communication port | -## Disk space requirements +## Storage requirements @@ -234,7 +255,7 @@ As an open-source distributed SQL database, TiDB requires the following network - + @@ -265,6 +286,37 @@ As an open-source distributed SQL database, TiDB requires the following network
TiDB
  • At least 30 GB for the log disk
  • Starting from v6.5.0, Fast Online DDL (controlled by the tidb_ddl_enable_fast_reorg variable) is enabled by default to accelerate DDL operations, such as adding indexes. If DDL operations involving large objects exist in your application, it is highly recommended to prepare additional SSD disk space for TiDB (100 GB or more). For detailed configuration instructions, see Set a temporary space for a TiDB instance
  • At least 30 GB for the log disk
  • Starting from v6.5.0, Fast Online DDL (controlled by the tidb_ddl_enable_fast_reorg variable) is enabled by default to accelerate DDL operations, such as adding indexes. If DDL operations involving large objects exist in your application, or you want to use IMPORT INTO to import data, it is highly recommended to prepare additional SSD disk space for TiDB (100 GB or more). For detailed configuration instructions, see Set a temporary space for a TiDB instance
Lower than 90%
+TiDB supports the XFS and Ext4 file systems. Other file systems are not recommended for production environments. + ## Web browser requirements TiDB relies on [Grafana](https://grafana.com/) to provide visualization of database metrics. A recent version of Internet Explorer, Chrome or Firefox with Javascript enabled is sufficient. + +## Hardware and software requirements for TiFlash disaggregated storage and compute architecture + +The preceding TiFlash software and hardware requirements are for the coupled storage and compute architecture. Starting from v7.0.0, TiFlash supports the [disaggregated storage and compute architecture](/tiflash/tiflash-disaggregated-and-s3.md). In this architecture, TiFlash is divided into two types of nodes: the Write Node and the Compute Node. The requirements for these nodes are as follows: + +- Software: remain the same as the coupled storage and compute architecture, see [OS and platform requirements](#os-and-platform-requirements). +- Network port: remain the same as the coupled storage and compute architecture, see [Network](#network-requirements). +- Disk space: + - TiFlash Write Node: it is recommended to configure at least 200 GB of disk space, which is used as a local buffer when adding TiFlash replicas and migrating Region replicas before uploading data to Amazon S3. In addition, an object storage compatible with Amazon S3 is required. + - TiFlash Compute Node: it is recommended to configure at least 100 GB of disk space, which is mainly used to cache the data read from the Write Node to improve performance. The cache of the Compute Node might be fully used, which is normal. +- CPU and memory requirements are described in the following sections. + +### Development and test environments + +| Component | CPU | Memory | Local Storage | Network | Number of Instances (Minimum Requirement) | +| --- | --- | --- | --- | --- | --- | +| TiFlash Write Node | 16 cores+ | 32 GB+ | SSD, 200 GB+ | Gigabit Ethernet | 1 | +| TiFlash Compute Node | 16 cores+ | 32 GB+ | SSD, 100 GB+ | Gigabit Ethernet | 0 (see the following note) | + +### Production environment + +| Component | CPU | Memory | Disk Type | Network | Number of Instances (Minimum Requirement) | +| --- | --- | --- | --- | --- | --- | +| TiFlash Write Node | 32 cores+ | 64 GB+ | 1 or more SSDs | 10 Gigabit Ethernet (2 recommended) | 1 | +| TiFlash Compute Node | 32 cores+ | 64 GB+ | 1 or more SSDs | 10 Gigabit Ethernet (2 recommended) | 0 (see the following note) | + +> **Note:** +> +> You can use deployment tools such as TiUP to quickly scale in or out the TiFlash Compute Node, within the range of `[0, +inf]`. diff --git a/hybrid-deployment-topology.md b/hybrid-deployment-topology.md index db2fe2028b687..0b55cc0678752 100644 --- a/hybrid-deployment-topology.md +++ b/hybrid-deployment-topology.md @@ -1,7 +1,6 @@ --- title: Hybrid Deployment Topology summary: Learn the hybrid deployment topology of TiDB clusters. -aliases: ['/docs/dev/hybrid-deployment-topology/'] --- # Hybrid Deployment Topology @@ -21,6 +20,10 @@ The deployment machine has multiple CPU processors with sufficient memory. To im | TiKV | 6 | 32 VCore 64GB | 10.0.1.7
10.0.1.8
10.0.1.9 | 1. Separate the instance-level port and status_port;
2. Configure the global parameters `readpool`, `storage` and `raftstore`;
3. Configure labels of the instance-level host;
4. Configure NUMA to bind CPU cores | | Monitoring & Grafana | 1 | 4 VCore 8GB * 1 500GB (ssd) | 10.0.1.10 | Default configuration | +> **Note:** +> +> The IP addresses of the instances are given as examples only. In your actual deployment, replace the IP addresses with your actual IP addresses. + ### Topology templates - [The simple template for the hybrid deployment](https://github.com/pingcap/docs/blob/master/config-templates/simple-multi-instance.yaml) diff --git a/identify-expensive-queries.md b/identify-expensive-queries.md index b9ea61935f970..c1b7919be26ce 100644 --- a/identify-expensive-queries.md +++ b/identify-expensive-queries.md @@ -1,6 +1,6 @@ --- title: Identify Expensive Queries -aliases: ['/docs/dev/identify-expensive-queries/','/docs/dev/how-to/maintain/identify-abnormal-queries/identify-expensive-queries/'] +summary: TiDB helps identify expensive queries by printing information about statements that exceed the execution time or memory usage threshold. This allows for diagnosing and improving SQL performance. The expensive query log includes details such as execution time, memory usage, user, database, and TiKV Coprocessor task information. This log differs from the slow query log as it prints information as soon as the statement exceeds the resource threshold. --- # Identify Expensive Queries diff --git a/identify-slow-queries.md b/identify-slow-queries.md index aaab48e497eeb..910f6fc334c0f 100644 --- a/identify-slow-queries.md +++ b/identify-slow-queries.md @@ -1,7 +1,6 @@ --- title: Identify Slow Queries summary: Use the slow query log to identify problematic SQL statements. -aliases: ['/docs/dev/identify-slow-queries/','/docs/dev/how-to/maintain/identify-abnormal-queries/identify-slow-queries/','/docs/dev/how-to/maintain/identify-slow-queries'] --- # Identify Slow Queries @@ -74,7 +73,7 @@ Slow query basics: * `Succ`: Whether a statement is executed successfully. * `Backoff_time`: The waiting time before retry when a statement encounters errors that require a retry. The common errors as such include: `lock occurs`, `Region split`, and `tikv server is busy`. * `Plan`: The execution plan of a statement. Execute the `SELECT tidb_decode_plan('xxx...')` statement to parse the specific execution plan. -* `Binary_plan`: The execution plan of a binary-encoded statement. Execute the `SELECT tidb_decode_binary_plan('xxx...')` statement to parse the specific execution plan. The `Plan` and `Binary_plan` fields carry the same information. However, the format of execution plans parsed from the two fields are different. +* `Binary_plan`: The execution plan of a binary-encoded statement. Execute the [`SELECT tidb_decode_binary_plan('xxx...')`](/functions-and-operators/tidb-functions.md#tidb_decode_binary_plan) statement to parse the specific execution plan. The `Plan` and `Binary_plan` fields carry the same information. However, the format of execution plans parsed from the two fields are different. * `Prepared`: Whether this statement is a `Prepare` or `Execute` request or not. * `Plan_from_cache`: Whether this statement hits the execution plan cache. * `Plan_from_binding`: Whether this statement uses the bound execution plans. @@ -161,6 +160,13 @@ TiKV Coprocessor Task fields: * `tikvDiskFull`: The backoff caused by that the TiKV disk is full. * `txnLockFast`: The backoff caused by that locks are encountered during data reads. +Fields related to Resource Control: + +* `Resource_group`: the resource group that the statement is bound to. +* `Request_unit_read`: the total read RUs consumed by the statement. +* `Request_unit_write`: the total write RUs consumed by the statement. +* `Time_queued_by_rc`: the total time that the statement waits for available resources. + ## Related system variables * [`tidb_slow_log_threshold`](/system-variables.md#tidb_slow_log_threshold): Sets the threshold for the slow log. The SQL statement whose execution time exceeds this threshold is recorded in the slow log. The default value is 300 (ms). @@ -607,7 +613,7 @@ The following table shows output details: | details | The details of the SQL execution | | succ | Whether the SQL statement is executed successfully. `1` means success and `0` means failure. | | conn_id | The connection ID for the session | -| transaction_ts | The `commit ts` for a transaction commit | +| transaction_ts | The `start ts` of the transaction | | user | The user name for the execution of the statement | | db | The database involved when the statement is executed | | table_ids | The ID of the table involved when the SQL statement is executed | diff --git a/import-example-data.md b/import-example-data.md index 8556ccda969fd..aba2472367787 100644 --- a/import-example-data.md +++ b/import-example-data.md @@ -1,7 +1,6 @@ --- title: Import Example Database summary: Install the Bikeshare example database. -aliases: ['/docs/dev/import-example-data/','/docs/dev/how-to/get-started/import-example-database/'] --- # Import Example Database diff --git a/information-schema/information-schema-cluster-config.md b/information-schema/information-schema-cluster-config.md index c5e1bfa1db225..dc414ab6bd901 100644 --- a/information-schema/information-schema-cluster-config.md +++ b/information-schema/information-schema-cluster-config.md @@ -1,7 +1,6 @@ --- title: CLUSTER_CONFIG summary: Learn the `CLUSTER_CONFIG` information_schema table. -aliases: ['/docs/dev/system-tables/system-table-cluster-config/','/docs/dev/reference/system-databases/cluster-config/','/tidb/dev/system-table-cluster-config/'] --- # CLUSTER_CONFIG @@ -10,7 +9,7 @@ You can use the `CLUSTER_CONFIG` cluster configuration table to get the current > **Note:** > -> This table is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This table is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). {{< copyable "sql" >}} diff --git a/information-schema/information-schema-cluster-hardware.md b/information-schema/information-schema-cluster-hardware.md index d3b1caa939e95..8232602b1b3c4 100644 --- a/information-schema/information-schema-cluster-hardware.md +++ b/information-schema/information-schema-cluster-hardware.md @@ -1,7 +1,6 @@ --- title: CLUSTER_HARDWARE summary: Learn the `CLUSTER_HARDWARE` information_schema table. -aliases: ['/docs/dev/system-tables/system-table-cluster-hardware/','/docs/dev/reference/system-databases/cluster-hardware/','/tidb/dev/system-table-cluster-hardware/'] --- # CLUSTER_HARDWARE @@ -10,7 +9,7 @@ The `CLUSTER_HARDWARE` hardware system table provides the hardware information o > **Note:** > -> This table is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This table is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). {{< copyable "sql" >}} diff --git a/information-schema/information-schema-cluster-info.md b/information-schema/information-schema-cluster-info.md index a1781a2f16016..e58ceac8b7475 100644 --- a/information-schema/information-schema-cluster-info.md +++ b/information-schema/information-schema-cluster-info.md @@ -1,7 +1,6 @@ --- title: CLUSTER_INFO summary: Learn the `CLUSTER_INFO` cluster topology information table. -aliases: ['/docs/dev/system-tables/system-table-cluster-info/','/docs/dev/reference/system-databases/cluster-info/','/tidb/dev/system-table-cluster-info/'] --- # CLUSTER_INFO @@ -10,7 +9,7 @@ The `CLUSTER_INFO` cluster topology table provides the current topology informat > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. {{< copyable "sql" >}} @@ -39,7 +38,7 @@ Field description: * `TYPE`: The instance type. The optional values are `tidb`, `pd`, and `tikv`. * `INSTANCE`: The instance address, which is a string in the format of `IP:PORT`. -* `STATUS_ADDRESS`: The service address of HTTP API. Some commands in tikv-ctl, pd-ctl, or tidb-ctl might use this API and this address. You can also get more cluster information via this address. Refer to [TiDB HTTP API document](https://github.com/pingcap/tidb/blob/master/docs/tidb_http_api.md) for details. +* `STATUS_ADDRESS`: The service address of HTTP API. Some commands in tikv-ctl, pd-ctl, or tidb-ctl might use this API and this address. You can also get more cluster information via this address. Refer to [TiDB HTTP API document](https://github.com/pingcap/tidb/blob/release-7.5/docs/tidb_http_api.md) for details. * `VERSION`: The semantic version number of the corresponding instance. To be compatible with the MySQL version number, the TiDB version is displayed in the format of `${mysql-version}-${tidb-version}`. * `GIT_HASH`: The Git Commit Hash when compiling the instance version, which is used to identify whether two instances are of the absolutely consistent version. * `START_TIME`: The starting time of the corresponding instance. diff --git a/information-schema/information-schema-cluster-load.md b/information-schema/information-schema-cluster-load.md index 21faa9a49ff41..69d2b1d173dc7 100644 --- a/information-schema/information-schema-cluster-load.md +++ b/information-schema/information-schema-cluster-load.md @@ -1,7 +1,6 @@ --- title: CLUSTER_LOAD summary: Learn the `CLUSTER_LOAD` information_schema table. -aliases: ['/docs/dev/system-tables/system-table-cluster-load/','/docs/dev/reference/system-databases/cluster-load/','/tidb/dev/system-table-cluster-load/'] --- # CLUSTER_LOAD @@ -10,7 +9,7 @@ The `CLUSTER_LOAD` cluster load table provides the current load information of t > **Note:** > -> This table is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This table is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). {{< copyable "sql" >}} diff --git a/information-schema/information-schema-cluster-log.md b/information-schema/information-schema-cluster-log.md index b9e7899dabd23..f9d94dd0d065e 100644 --- a/information-schema/information-schema-cluster-log.md +++ b/information-schema/information-schema-cluster-log.md @@ -1,7 +1,6 @@ --- title: CLUSTER_LOG summary: Learn the `CLUSTER_LOG` information_schema table. -aliases: ['/docs/dev/system-tables/system-table-cluster-log/','/docs/dev/reference/system-databases/cluster-log/','/tidb/dev/system-table-cluster-log/'] --- # CLUSTER_LOG @@ -10,7 +9,7 @@ You can query cluster logs on the `CLUSTER_LOG` cluster log table. By pushing do > **Note:** > -> This table is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This table is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). To get the logs of the TiDB cluster before v4.0, you need to log in to each instance to summarize logs. This cluster log table in 4.0 provides the global and time-ordered log search result, which makes it easier to track full-link events. For example, by searching logs according to the `region id`, you can query all logs in the life cycle of this Region. Similarly, by searching the full link log through the slow log's `txn id`, you can query the flow and the number of keys scanned by this transaction at each instance. diff --git a/information-schema/information-schema-cluster-systeminfo.md b/information-schema/information-schema-cluster-systeminfo.md index 2f84af2406f62..c915561fb98d0 100644 --- a/information-schema/information-schema-cluster-systeminfo.md +++ b/information-schema/information-schema-cluster-systeminfo.md @@ -1,7 +1,6 @@ --- title: CLUSTER_SYSTEMINFO summary: Learn the `CLUSTER_SYSTEMINFO` kernel parameter table. -aliases: ['/docs/dev/system-tables/system-table-cluster-systeminfo/','/docs/dev/reference/system-databases/cluster-systeminfo/','/tidb/dev/system-table-cluster-systeminfo/'] --- # CLUSTER_SYSTEMINFO @@ -10,7 +9,7 @@ You can use the `CLUSTER_SYSTEMINFO` kernel parameter table to query the kernel > **Note:** > -> This table is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This table is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). {{< copyable "sql" >}} diff --git a/information-schema/information-schema-columns.md b/information-schema/information-schema-columns.md index 7535816ee6f73..5c675bf4a56d8 100644 --- a/information-schema/information-schema-columns.md +++ b/information-schema/information-schema-columns.md @@ -50,7 +50,7 @@ CREATE TABLE test.t1 (a int); SELECT * FROM COLUMNS WHERE table_schema='test' AND TABLE_NAME='t1'\G ``` -输出结果如下: +The output is as follows: ```sql *************************** 1. row *************************** diff --git a/information-schema/information-schema-data-lock-waits.md b/information-schema/information-schema-data-lock-waits.md index c6a31fb4a315f..9b98df5b99fd3 100644 --- a/information-schema/information-schema-data-lock-waits.md +++ b/information-schema/information-schema-data-lock-waits.md @@ -88,4 +88,4 @@ CURRENT_HOLDING_TRX_ID: 426790590082449409 1 row in set (0.01 sec) ``` -The above query result shows that the transaction of the ID `426790594290122753` is trying to obtain the pessimistic lock on the key `"7480000000000000355F728000000000000001"` when executing a statement that has digest `"38b03afa5debbdf0326a014dbe5012a62c51957f1982b3093e748460f8b00821"` and is in the form of ``update `t` set `v` = `v` + ? where `id` = ?``, but the lock on this key was held by the transaction of the ID `426790590082449409`. +The above query result shows that the transaction of the ID `426790594290122753` is trying to obtain the pessimistic lock on the key `"7480000000000000355F728000000000000001"` when executing a statement that has digest `"38b03afa5debbdf0326a014dbe5012a62c51957f1982b3093e748460f8b00821"` and is in the form of ``update `t` set `v` = `v` + ? where `id` = ?``, but the lock on this key was held by the transaction of the ID `426790590082449409`. diff --git a/information-schema/information-schema-ddl-jobs.md b/information-schema/information-schema-ddl-jobs.md index 67b38c2d68ca6..21f4b69ae5304 100644 --- a/information-schema/information-schema-ddl-jobs.md +++ b/information-schema/information-schema-ddl-jobs.md @@ -5,7 +5,7 @@ summary: Learn the `DDL_JOBS` information_schema table. # DDL_JOBS -The `DDL_JOBS` table provides an `INFORMATION_SCHEMA` interface to the [`ADMIN SHOW DDL JOBS`](/sql-statements/sql-statement-admin-show-ddl.md) command. It provides both the current status and a short history of DDL operations across the TiDB cluster. +The `DDL_JOBS` table provides an `INFORMATION_SCHEMA` interface to the [`ADMIN SHOW DDL JOBS`](/sql-statements/sql-statement-admin-show-ddl.md) command. It provides information about DDL operations in the TiDB cluster, such as the current status, DDL statements, start time, end time, database names, and table names. {{< copyable "sql" >}} diff --git a/information-schema/information-schema-deadlocks.md b/information-schema/information-schema-deadlocks.md index e755d59cda0f9..debbeb65834c2 100644 --- a/information-schema/information-schema-deadlocks.md +++ b/information-schema/information-schema-deadlocks.md @@ -12,7 +12,7 @@ USE INFORMATION_SCHEMA; DESC deadlocks; ``` -Thhe output is as follows: +The output is as follows: ```sql +-------------------------+---------------------+------+------+---------+-------+ diff --git a/information-schema/information-schema-inspection-result.md b/information-schema/information-schema-inspection-result.md index ab0ed11c581d6..2ccffb8b5ec61 100644 --- a/information-schema/information-schema-inspection-result.md +++ b/information-schema/information-schema-inspection-result.md @@ -1,7 +1,6 @@ --- title: INSPECTION_RESULT summary: Learn the `INSPECTION_RESULT` diagnostic result table. -aliases: ['/docs/dev/system-tables/system-table-inspection-result/','/docs/dev/reference/system-databases/inspection-result/','/tidb/dev/system-table-inspection-result/'] --- # INSPECTION_RESULT @@ -12,7 +11,7 @@ The `INSPECTION_RESULT` diagnostic table can help you quickly find problems and > **Note:** > -> This table is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This table is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). The structure of the `information_schema.inspection_result` diagnostic result table `information_schema.inspection_result` is as follows: @@ -264,7 +263,7 @@ In `critical-error` diagnostic rule, the following two diagnostic rules are exec | ---- | ---- | ---- | ---- | | TiDB | panic-count | tidb_panic_count_total_count | Panic occurs in TiDB. | | TiDB | binlog-error | tidb_binlog_error_total_count | An error occurs when TiDB writes binlog. | - | TiKV | critical-error | tikv_critical_error_total_coun | The critical error of TiKV. | + | TiKV | critical-error | tikv_critical_error_total_count | The critical error of TiKV. | | TiKV | scheduler-is-busy | tikv_scheduler_is_busy_total_count | The TiKV scheduler is too busy, which makes TiKV temporarily unavailable. | | TiKV | coprocessor-is-busy | tikv_coprocessor_is_busy_total_count | The TiKV Coprocessor is too busy. | | TiKV | channel-is-full | tikv_channel_full_total_count | The "channel full" error occurs in TiKV. | diff --git a/information-schema/information-schema-inspection-rules.md b/information-schema/information-schema-inspection-rules.md index fec49f99ad45a..15f3e9b49b582 100644 --- a/information-schema/information-schema-inspection-rules.md +++ b/information-schema/information-schema-inspection-rules.md @@ -9,7 +9,7 @@ The `INSPECTION_RULES` table provides information about which diagnostic tests a > **Note:** > -> This table is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This table is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). {{< copyable "sql" >}} diff --git a/information-schema/information-schema-inspection-summary.md b/information-schema/information-schema-inspection-summary.md index 0ba9686b0d397..2ccfcaa001ae0 100644 --- a/information-schema/information-schema-inspection-summary.md +++ b/information-schema/information-schema-inspection-summary.md @@ -1,7 +1,6 @@ --- title: INSPECTION_SUMMARY summary: Learn the `INSPECTION_SUMMARY` inspection summary table. -aliases: ['/docs/dev/system-tables/system-table-inspection-summary/','/docs/dev/reference/system-databases/inspection-summary/','/tidb/dev/system-table-inspection-summary/'] --- # INSPECTION_SUMMARY @@ -10,7 +9,7 @@ In some scenarios, you might need to pay attention only to the monitoring summar > **Note:** > -> This table is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This table is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). The structure of the `information_schema.inspection_summary` inspection summary table is as follows: diff --git a/information-schema/information-schema-keywords.md b/information-schema/information-schema-keywords.md new file mode 100644 index 0000000000000..b3d4daa733829 --- /dev/null +++ b/information-schema/information-schema-keywords.md @@ -0,0 +1,48 @@ +--- +title: KEYWORDS +summary: Learn the `KEYWORDS` INFORMATION_SCHEMA table. +--- + +# KEYWORDS + +Starting from v7.5.3, TiDB provides the `KEYWORDS` table. You can use this table to get information about [keywords](/keywords.md) in TiDB. + +```sql +USE INFORMATION_SCHEMA; +DESC keywords; +``` + +The output is as follows: + +``` ++----------+--------------+------+------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++----------+--------------+------+------+---------+-------+ +| WORD | varchar(128) | YES | | NULL | | +| RESERVED | int(11) | YES | | NULL | | ++----------+--------------+------+------+---------+-------+ +2 rows in set (0.00 sec) +``` + +Field description: + +- `WORD`: The keyword. +- `RESERVED`: Whether the keyword is reserved. + +The following statement queries the information about `ADD` and `USER` keywords: + +```sql +SELECT * FROM INFORMATION_SCHEMA.KEYWORDS WHERE WORD IN ('ADD','USER'); +``` + +From the output, you can see that `ADD` is a reserved keyword and `USER` is a non-reserved keyword. + +``` ++------+----------+ +| WORD | RESERVED | ++------+----------+ +| ADD | 1 | +| USER | 0 | ++------+----------+ +2 rows in set (0.00 sec) +``` diff --git a/information-schema/information-schema-metrics-summary.md b/information-schema/information-schema-metrics-summary.md index e8bc9d5aeb9e5..9f9f2639a6be9 100644 --- a/information-schema/information-schema-metrics-summary.md +++ b/information-schema/information-schema-metrics-summary.md @@ -1,7 +1,6 @@ --- title: METRICS_SUMMARY summary: Learn the METRICS_SUMMARY system table. -aliases: ['/docs/dev/system-tables/system-table-metrics-summary/','/docs/dev/reference/system-databases/metrics-summary/','/tidb/dev/system-table-metrics-summary'] --- # METRICS_SUMMARY @@ -13,7 +12,7 @@ The TiDB cluster has many monitoring metrics. To make it easy to detect abnormal > **Note:** > -> The preceding two monitoring summary tables are only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> The preceding two monitoring summary tables are only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). The two tables summarize all monitoring data for you to check each monitoring metric efficiently. Compared with `information_schema.metrics_summary`, the `information_schema.metrics_summary_by_label` table has an additional `label` column and performs differentiated statistics according to different labels. diff --git a/information-schema/information-schema-metrics-tables.md b/information-schema/information-schema-metrics-tables.md index 9d3226bc40d9d..9a31e8407814a 100644 --- a/information-schema/information-schema-metrics-tables.md +++ b/information-schema/information-schema-metrics-tables.md @@ -1,7 +1,6 @@ --- title: METRICS_TABLES summary: Learn the `METRICS_TABLES` system table. -aliases: ['/docs/dev/system-tables/system-table-metrics-tables/','/docs/dev/reference/system-databases/metrics-tables/','/tidb/dev/system-table-metrics-tables/'] --- # METRICS_TABLES @@ -10,7 +9,7 @@ The `METRICS_TABLES` table provides the PromQL (Prometheus Query Language) defin > **Note:** > -> This table is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This table is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). ```sql USE INFORMATION_SCHEMA; diff --git a/information-schema/information-schema-placement-policies.md b/information-schema/information-schema-placement-policies.md index 59130a1896194..4fac962198c94 100644 --- a/information-schema/information-schema-placement-policies.md +++ b/information-schema/information-schema-placement-policies.md @@ -1,7 +1,6 @@ --- title: PLACEMENT_POLICIES summary: Learn the `PLACEMENT_POLICIES` information_schema table. -aliases: ['/tidb/dev/information-schema-placement-rules'] --- # PLACEMENT_POLICIES @@ -10,7 +9,7 @@ The `PLACEMENT_POLICIES` table provides information on all placement policies. F > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. {{< copyable "sql" >}} diff --git a/information-schema/information-schema-resource-groups.md b/information-schema/information-schema-resource-groups.md index e2b86a07d2f91..592386dc32189 100644 --- a/information-schema/information-schema-resource-groups.md +++ b/information-schema/information-schema-resource-groups.md @@ -9,7 +9,7 @@ The `RESOURCE_GROUPS` table shows the information about all resource groups. For > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ```sql USE information_schema; diff --git a/information-schema/information-schema-runaway-watches.md b/information-schema/information-schema-runaway-watches.md index dfebf661e9a49..d247ae9119939 100644 --- a/information-schema/information-schema-runaway-watches.md +++ b/information-schema/information-schema-runaway-watches.md @@ -7,6 +7,10 @@ summary: Learn the `RUNAWAY_WATCHES` INFORMATION_SCHEMA table. The `RUNAWAY_WATCHES` table shows the watch list of runaway queries that consume more resources than expected. For more information, see [Runaway Queries](/tidb-resource-control.md#manage-queries-that-consume-more-resources-than-expected-runaway-queries). +> **Note:** +> +> This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. + ```sql USE INFORMATION_SCHEMA; DESC RUNAWAY_WATCHES; @@ -37,7 +41,7 @@ DESC RUNAWAY_WATCHES; Query the watch list of runaway queries: ```sql -SELECT * FROM INFORMATION_SCHEMA.RUNAWAY_WATCHES; +SELECT * FROM INFORMATION_SCHEMA.RUNAWAY_WATCHES\G ``` The output is as follows: @@ -73,7 +77,7 @@ QUERY WATCH ADD RESOURCE GROUP rg1 SQL TEXT EXACT TO 'select * from sbtest.sbtes Query the watch list of runaway queries again: ```sql -SELECT * FROM INFORMATION_SCHEMA.RUNAWAY_WATCHES\G; +SELECT * FROM INFORMATION_SCHEMA.RUNAWAY_WATCHES\G ``` The output is as follows: diff --git a/information-schema/information-schema-sequences.md b/information-schema/information-schema-sequences.md index 60a0cb408e2ff..f7e92db6ae99a 100644 --- a/information-schema/information-schema-sequences.md +++ b/information-schema/information-schema-sequences.md @@ -37,7 +37,7 @@ Create a sequence `test.seq` and query the next value of the sequence: ```sql CREATE SEQUENCE test.seq; -SELECT nextval(test.seq); +SELECT NEXTVAL(test.seq); SELECT * FROM sequences\G ``` @@ -45,7 +45,7 @@ The output is as follows: ```sql +-------------------+ -| nextval(test.seq) | +| NEXTVAL(test.seq) | +-------------------+ | 1 | +-------------------+ diff --git a/information-schema/information-schema-slow-query.md b/information-schema/information-schema-slow-query.md index 8510aec12ffc5..b139323e41a1e 100644 --- a/information-schema/information-schema-slow-query.md +++ b/information-schema/information-schema-slow-query.md @@ -9,7 +9,7 @@ The `SLOW_QUERY` table provides the slow query information of the current node, > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. @@ -24,7 +24,7 @@ DESC SLOW_QUERY; The output is as follows: -```sqlsql +``` +-------------------------------+---------------------+------+------+---------+-------+ | Field | Type | Null | Key | Default | Extra | +-------------------------------+---------------------+------+------+---------+-------+ @@ -33,6 +33,7 @@ The output is as follows: | User | varchar(64) | YES | | NULL | | | Host | varchar(64) | YES | | NULL | | | Conn_ID | bigint(20) unsigned | YES | | NULL | | +| Session_alias | varchar(64) | YES | | NULL | | | Exec_retry_count | bigint(20) unsigned | YES | | NULL | | | Exec_retry_time | double | YES | | NULL | | | Query_time | double | YES | | NULL | | @@ -97,22 +98,28 @@ The output is as follows: | Plan_from_cache | tinyint(1) | YES | | NULL | | | Plan_from_binding | tinyint(1) | YES | | NULL | | | Has_more_results | tinyint(1) | YES | | NULL | | +| Resource_group | varchar(64) | YES | | NULL | | +| Request_unit_read | double | YES | | NULL | | +| Request_unit_write | double | YES | | NULL | | +| Time_queued_by_rc | double | YES | | NULL | | | Plan | longtext | YES | | NULL | | | Plan_digest | varchar(128) | YES | | NULL | | | Binary_plan | longtext | YES | | NULL | | | Prev_stmt | longtext | YES | | NULL | | | Query | longtext | YES | | NULL | | +-------------------------------+---------------------+------+------+---------+-------+ -74 rows in set (0.001 sec) +79 rows in set (0.00 sec) ``` +The maximum statement length of the `Query` column is limited by the [`tidb_stmt_summary_max_sql_length`](/system-variables.md#tidb_stmt_summary_max_sql_length-new-in-v40) system variable. + ## CLUSTER_SLOW_QUERY table The `CLUSTER_SLOW_QUERY` table provides the slow query information of all nodes in the cluster, which is the parsing result of the TiDB slow log files. You can use the `CLUSTER_SLOW_QUERY` table the way you do with `SLOW_QUERY`. The table schema of the `CLUSTER_SLOW_QUERY` table differs from that of the `SLOW_QUERY` table in that an `INSTANCE` column is added to `CLUSTER_SLOW_QUERY`. The `INSTANCE` column represents the TiDB node address of the row information on the slow query. > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. @@ -136,6 +143,7 @@ The output is as follows: | User | varchar(64) | YES | | NULL | | | Host | varchar(64) | YES | | NULL | | | Conn_ID | bigint(20) unsigned | YES | | NULL | | +| Session_alias | varchar(64) | YES | | NULL | | | Exec_retry_count | bigint(20) unsigned | YES | | NULL | | | Exec_retry_time | double | YES | | NULL | | | Query_time | double | YES | | NULL | | @@ -200,13 +208,17 @@ The output is as follows: | Plan_from_cache | tinyint(1) | YES | | NULL | | | Plan_from_binding | tinyint(1) | YES | | NULL | | | Has_more_results | tinyint(1) | YES | | NULL | | +| Resource_group | varchar(64) | YES | | NULL | | +| Request_unit_read | double | YES | | NULL | | +| Request_unit_write | double | YES | | NULL | | +| Time_queued_by_rc | double | YES | | NULL | | | Plan | longtext | YES | | NULL | | | Plan_digest | varchar(128) | YES | | NULL | | | Binary_plan | longtext | YES | | NULL | | | Prev_stmt | longtext | YES | | NULL | | | Query | longtext | YES | | NULL | | +-------------------------------+---------------------+------+------+---------+-------+ -75 rows in set (0.001 sec) +80 rows in set (0.00 sec) ``` When the cluster system table is queried, TiDB does not obtain data from all nodes, but pushes down the related calculation to other nodes. The execution plan is as follows: @@ -222,9 +234,9 @@ The output is as follows: | id | estRows | task | access object | operator info | +----------------------------+----------+-----------+--------------------------+------------------------------------------------------+ | StreamAgg_7 | 1.00 | root | | funcs:count(1)->Column#75 | -| └─TableReader_13 | 10.00 | root | | data:Selection_12 | -| └─Selection_12 | 10.00 | cop[tidb] | | eq(INFORMATION_SCHEMA.cluster_slow_query.user, "u1") | -| └─TableFullScan_11 | 10000.00 | cop[tidb] | table:CLUSTER_SLOW_QUERY | keep order:false, stats:pseudo | +| └─TableReader_13 | 10.00 | root | | data:Selection_12 | +| └─Selection_12 | 10.00 | cop[tidb] | | eq(INFORMATION_SCHEMA.cluster_slow_query.user, "u1") | +| └─TableFullScan_11 | 10000.00 | cop[tidb] | table:CLUSTER_SLOW_QUERY | keep order:false, stats:pseudo | +----------------------------+----------+-----------+--------------------------+------------------------------------------------------+ 4 rows in set (0.00 sec) ``` diff --git a/information-schema/information-schema-sql-diagnostics.md b/information-schema/information-schema-sql-diagnostics.md index ab65d80a104df..828a800af8985 100644 --- a/information-schema/information-schema-sql-diagnostics.md +++ b/information-schema/information-schema-sql-diagnostics.md @@ -1,7 +1,6 @@ --- title: SQL Diagnostics summary: Understand SQL diagnostics in TiDB. -aliases: ['/docs/dev/system-tables/system-table-sql-diagnostics/','/docs/dev/reference/system-databases/sql-diagnosis/','/docs/dev/system-tables/system-table-sql-diagnosis/','/tidb/dev/system-table-sql-diagnostics/','/tidb/dev/check-cluster-status-using-sql-statements','/docs/dev/check-cluster-status-using-sql-statements/','/docs/dev/reference/performance/check-cluster-status-using-sql-statements/'] --- # SQL Diagnostics diff --git a/information-schema/information-schema-tidb-hot-regions-history.md b/information-schema/information-schema-tidb-hot-regions-history.md index 314a8091be243..239d0de3e66e5 100644 --- a/information-schema/information-schema-tidb-hot-regions-history.md +++ b/information-schema/information-schema-tidb-hot-regions-history.md @@ -9,7 +9,7 @@ The `TIDB_HOT_REGIONS_HISTORY` table provides information about history hot Regi > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. diff --git a/information-schema/information-schema-tidb-hot-regions.md b/information-schema/information-schema-tidb-hot-regions.md index 60084f3ca4218..127cc9188d8e1 100644 --- a/information-schema/information-schema-tidb-hot-regions.md +++ b/information-schema/information-schema-tidb-hot-regions.md @@ -9,7 +9,7 @@ The `TIDB_HOT_REGIONS` table provides information about the current hot Regions. > **Note:** > -> This table is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This table is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). {{< copyable "sql" >}} diff --git a/information-schema/information-schema-tidb-servers-info.md b/information-schema/information-schema-tidb-servers-info.md index cc56d0a1ecc2b..f8c026e6c7881 100644 --- a/information-schema/information-schema-tidb-servers-info.md +++ b/information-schema/information-schema-tidb-servers-info.md @@ -9,7 +9,7 @@ The `TIDB_SERVERS_INFO` table provides information about TiDB servers in the TiD > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ```sql USE INFORMATION_SCHEMA; @@ -50,7 +50,7 @@ The output is as follows: PORT: 4000 STATUS_PORT: 10080 LEASE: 45s - VERSION: 8.0.11-TiDB-v7.4.0 + VERSION: 8.0.11-TiDB-v{{{ .tidb-version }}} GIT_HASH: 827d8ff2d22ac4c93ae1b841b79d468211e1d393 BINLOG_STATUS: Off LABELS: diff --git a/information-schema/information-schema-tiflash-segments.md b/information-schema/information-schema-tiflash-segments.md index d3d7f585d12c1..54ca2a18f82e9 100644 --- a/information-schema/information-schema-tiflash-segments.md +++ b/information-schema/information-schema-tiflash-segments.md @@ -9,10 +9,6 @@ summary: Learn the `TIFLASH_SEGMENTS` information_schema table. > > Do not use this table in production environments, as the fields of the table are unstable, and subject to change in new releases of TiDB, without prior notice. -> **Note:** -> -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. - The `TIFLASH_SEGMENTS` table provides statistical information about data tables in TiFlash. ```sql diff --git a/information-schema/information-schema-tiflash-tables.md b/information-schema/information-schema-tiflash-tables.md index f7971996cb86c..42992d07809c5 100644 --- a/information-schema/information-schema-tiflash-tables.md +++ b/information-schema/information-schema-tiflash-tables.md @@ -9,10 +9,6 @@ summary: Learn the `TIFLASH_TABLES` information_schema table. > > Do not use this table in production environments, as the fields of the table are unstable, and subject to change in new releases of TiDB, without prior notice. -> **Note:** -> -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. - The `TIFLASH_TABLES` table provides statistical information about data tables in TiFlash. ```sql diff --git a/information-schema/information-schema-tikv-region-peers.md b/information-schema/information-schema-tikv-region-peers.md index c9018c20caeb2..f685f17ee230b 100644 --- a/information-schema/information-schema-tikv-region-peers.md +++ b/information-schema/information-schema-tikv-region-peers.md @@ -9,7 +9,7 @@ The `TIKV_REGION_PEERS` table shows detailed information of a single Region node > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ```sql USE INFORMATION_SCHEMA; diff --git a/information-schema/information-schema-tikv-region-status.md b/information-schema/information-schema-tikv-region-status.md index f9d225ee66b5d..f313136b0d5b8 100644 --- a/information-schema/information-schema-tikv-region-status.md +++ b/information-schema/information-schema-tikv-region-status.md @@ -9,7 +9,7 @@ The `TIKV_REGION_STATUS` table shows some basic information of TiKV Regions via > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. {{< copyable "sql" >}} diff --git a/information-schema/information-schema-tikv-store-status.md b/information-schema/information-schema-tikv-store-status.md index daee44e63533d..808a6a57a8a49 100644 --- a/information-schema/information-schema-tikv-store-status.md +++ b/information-schema/information-schema-tikv-store-status.md @@ -9,7 +9,7 @@ The `TIKV_STORE_STATUS` table shows some basic information of TiKV nodes via PD' > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ```sql USE INFORMATION_SCHEMA; diff --git a/information-schema/information-schema-user-attributes.md b/information-schema/information-schema-user-attributes.md index c2d31494a3819..9cc1b06662dca 100644 --- a/information-schema/information-schema-user-attributes.md +++ b/information-schema/information-schema-user-attributes.md @@ -5,7 +5,7 @@ summary: Learn the `USER_ATTRIBUTES` INFORMATION_SCHEMA table. # USER_ATTRIBUTES -The `USER_PRIVILEGES` table provides information about user comments and user attributes. This information comes from the `mysql.user` system table. +The `USER_ATTRIBUTES` table provides information about user comments and user attributes. This information comes from the `mysql.user` system table. ```sql USE information_schema; diff --git a/information-schema/information-schema-user-privileges.md b/information-schema/information-schema-user-privileges.md index 06c8e1507a9d6..9d2a835f28788 100644 --- a/information-schema/information-schema-user-privileges.md +++ b/information-schema/information-schema-user-privileges.md @@ -79,7 +79,7 @@ The output is as follows: - + ```sql +------------+---------------+-------------------------+--------------+ diff --git a/information-schema/information-schema-variables-info.md b/information-schema/information-schema-variables-info.md index 42c6cc8e7a6cb..6ed4dc7bec579 100644 --- a/information-schema/information-schema-variables-info.md +++ b/information-schema/information-schema-variables-info.md @@ -46,7 +46,7 @@ SELECT * FROM variables_info ORDER BY variable_name LIMIT 3; Fields in the `VARIABLES_INFO` table are described as follows: * `VARIABLE_NAME`: the name of the system variable. -* `VARIABLE_SCOPE`: the scope of the system variable. `SESSION` means that the system variable is only valid in the current session. `INSTANCE` means that the system variable is valid in the TiDB instance. `GLOBAL` means that the system variable is valid in the TiDB cluster. +* `VARIABLE_SCOPE`: the scope of the system variable. `SESSION` means that the system variable is only valid in the current session. `INSTANCE` means that the system variable is valid in the TiDB instance. `GLOBAL` means that the system variable is valid in the TiDB cluster. `NONE` means that the system variable is read only in the TiDB cluster. * `DEFAULT_VALUE`: the default value of the system variable. * `CURRENT_VALUE`: the current value of the system variable. If the scope includes `SESSION`, `CURRENT_VALUE` is the value in the current session. * `MIN_VALUE`: the minimum value allowed for the system variable. If the system variable is not numeric, `MIN_VALUE` is NULL. diff --git a/information-schema/information-schema.md b/information-schema/information-schema.md index 4994d48ba4319..be58588a8309c 100644 --- a/information-schema/information-schema.md +++ b/information-schema/information-schema.md @@ -1,7 +1,6 @@ --- title: Information Schema summary: TiDB implements the ANSI-standard information_schema for viewing system metadata. -aliases: ['/docs/dev/system-tables/system-table-information-schema/','/docs/dev/reference/system-databases/information-schema/','/tidb/dev/system-table-information-schema/'] --- # Information Schema @@ -28,6 +27,7 @@ Many `INFORMATION_SCHEMA` tables have a corresponding `SHOW` command. The benefi | `FILES` | Not implemented by TiDB. Returns zero rows. | | `GLOBAL_STATUS` | Not implemented by TiDB. Returns zero rows. | | `GLOBAL_VARIABLES` | Not implemented by TiDB. Returns zero rows. | +| [`KEYWORDS`](/information-schema/information-schema-keywords.md) | Provides a full list of keywords. | | [`KEY_COLUMN_USAGE`](/information-schema/information-schema-key-column-usage.md) | Describes the key constraints of the columns, such as the primary key constraint. | | `OPTIMIZER_TRACE` | Not implemented by TiDB. Returns zero rows. | | `PARAMETERS` | Not implemented by TiDB. Returns zero rows. | @@ -70,6 +70,7 @@ Many `INFORMATION_SCHEMA` tables have a corresponding `SHOW` command. The benefi | `FILES` | Not implemented by TiDB. Returns zero rows. | | `GLOBAL_STATUS` | Not implemented by TiDB. Returns zero rows. | | `GLOBAL_VARIABLES` | Not implemented by TiDB. Returns zero rows. | +| [`KEYWORDS`](/information-schema/information-schema-keywords.md) | Provides a full list of keywords. | | [`KEY_COLUMN_USAGE`](/information-schema/information-schema-key-column-usage.md) | Describes the key constraints of the columns, such as the primary key constraint. | | `OPTIMIZER_TRACE` | Not implemented by TiDB. Returns zero rows. | | `PARAMETERS` | Not implemented by TiDB. Returns zero rows. | @@ -102,7 +103,7 @@ Many `INFORMATION_SCHEMA` tables have a corresponding `SHOW` command. The benefi > **Note:** > -> Some of the following tables are only supported on TiDB Self-Hosted and not supported on TiDB Cloud. To get a full list of unsupported tables on TiDB Cloud, see [System tables](https://docs.pingcap.com/tidbcloud/limited-sql-features#system-tables). +> Some of the following tables are only supported on TiDB Self-Managed and not supported on TiDB Cloud. To get a full list of unsupported tables on TiDB Cloud, see [System tables](https://docs.pingcap.com/tidbcloud/limited-sql-features#system-tables). | Table Name | Description | |-----------------------------------------------------------------------------------------|-------------| @@ -113,15 +114,16 @@ Many `INFORMATION_SCHEMA` tables have a corresponding `SHOW` command. The benefi | [`CLUSTER_CONFIG`](/information-schema/information-schema-cluster-config.md) | Provides details about configuration settings for the entire TiDB cluster. This table is not applicable to TiDB Cloud. | | `CLUSTER_DEADLOCKS` | Provides a cluster-level view of the `DEADLOCKS` table. | | [`CLUSTER_HARDWARE`](/information-schema/information-schema-cluster-hardware.md) | Provides details on the underlying physical hardware discovered on each TiDB component. This table is not applicable to TiDB Cloud. | -| [`CLUSTER_INFO`](/information-schema/information-schema-cluster-info.md) | Provides details on the current cluster topology. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | +| [`CLUSTER_INFO`](/information-schema/information-schema-cluster-info.md) | Provides details on the current cluster topology. This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. | | [`CLUSTER_LOAD`](/information-schema/information-schema-cluster-load.md) | Provides current load information for TiDB servers in the cluster. This table is not applicable to TiDB Cloud. | | [`CLUSTER_LOG`](/information-schema/information-schema-cluster-log.md) | Provides a log for the entire TiDB cluster. This table is not applicable to TiDB Cloud. | | `CLUSTER_MEMORY_USAGE` | Provides a cluster-level view of the `MEMORY_USAGE` table. | | `CLUSTER_MEMORY_USAGE_OPS_HISTORY` | Provides a cluster-level view of the `MEMORY_USAGE_OPS_HISTORY` table. | | `CLUSTER_PROCESSLIST` | Provides a cluster-level view of the `PROCESSLIST` table. | -| `CLUSTER_SLOW_QUERY` | Provides a cluster-level view of the `SLOW_QUERY` table. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | -| `CLUSTER_STATEMENTS_SUMMARY` | Provides a cluster-level view of the `STATEMENTS_SUMMARY` table. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | -| `CLUSTER_STATEMENTS_SUMMARY_HISTORY` | Provides a cluster-level view of the `STATEMENTS_SUMMARY_HISTORY` table. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | +| `CLUSTER_SLOW_QUERY` | Provides a cluster-level view of the `SLOW_QUERY` table. This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. | +| `CLUSTER_STATEMENTS_SUMMARY` | Provides a cluster-level view of the `STATEMENTS_SUMMARY` table. This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. | +| `CLUSTER_STATEMENTS_SUMMARY_HISTORY` | Provides a cluster-level view of the `STATEMENTS_SUMMARY_HISTORY` table. This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. | +| `CLUSTER_TIDB_INDEX_USAGE` | Provides a cluster-level view of the `TIDB_INDEX_USAGE` table. | | `CLUSTER_TIDB_TRX` | Provides a cluster-level view of the `TIDB_TRX` table. | | [`CLUSTER_SYSTEMINFO`](/information-schema/information-schema-cluster-systeminfo.md) | Provides details about kernel parameter configuration for servers in the cluster. This table is not applicable to TiDB Cloud. | | [`DATA_LOCK_WAITS`](/information-schema/information-schema-data-lock-waits.md) | Provides the lock-waiting information on the TiKV server. | @@ -135,11 +137,11 @@ Many `INFORMATION_SCHEMA` tables have a corresponding `SHOW` command. The benefi | [`METRICS_SUMMARY`](/information-schema/information-schema-metrics-summary.md) | A summary of metrics extracted from Prometheus. This table is not applicable to TiDB Cloud. | | `METRICS_SUMMARY_BY_LABEL` | See `METRICS_SUMMARY` table. This table is not applicable to TiDB Cloud. | | [`METRICS_TABLES`](/information-schema/information-schema-metrics-tables.md) | Provides the PromQL definitions for tables in `METRICS_SCHEMA`. This table is not applicable to TiDB Cloud. | -| [`PLACEMENT_POLICIES`](/information-schema/information-schema-placement-policies.md) | Provides information on all placement policies. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | +| [`PLACEMENT_POLICIES`](/information-schema/information-schema-placement-policies.md) | Provides information on all placement policies. This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. | | [`SEQUENCES`](/information-schema/information-schema-sequences.md) | The TiDB implementation of sequences is based on MariaDB. | -| [`SLOW_QUERY`](/information-schema/information-schema-slow-query.md) | Provides information on slow queries on the current TiDB server. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | -| [`STATEMENTS_SUMMARY`](/statement-summary-tables.md) | Similar to performance_schema statement summary in MySQL. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | -| [`STATEMENTS_SUMMARY_HISTORY`](/statement-summary-tables.md) | Similar to performance_schema statement summary history in MySQL. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | +| [`SLOW_QUERY`](/information-schema/information-schema-slow-query.md) | Provides information on slow queries on the current TiDB server. This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. | +| [`STATEMENTS_SUMMARY`](/statement-summary-tables.md) | Similar to performance_schema statement summary in MySQL. This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. | +| [`STATEMENTS_SUMMARY_HISTORY`](/statement-summary-tables.md) | Similar to performance_schema statement summary history in MySQL. This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. | | [`TABLE_STORAGE_STATS`](/information-schema/information-schema-table-storage-stats.md) | Provides details about table sizes in storage. | | [`TIDB_HOT_REGIONS`](/information-schema/information-schema-tidb-hot-regions.md) | Provides statistics about which regions are hot. | | [`TIDB_HOT_REGIONS_HISTORY`](/information-schema/information-schema-tidb-hot-regions-history.md) | Provides history statistics about which Regions are hot. | @@ -164,15 +166,15 @@ Many `INFORMATION_SCHEMA` tables have a corresponding `SHOW` command. The benefi | [`CLUSTER_CONFIG`](https://docs.pingcap.com/tidb/stable/information-schema-cluster-config) | Provides details about configuration settings for the entire TiDB cluster. This table is not applicable to TiDB Cloud. | | `CLUSTER_DEADLOCKS` | Provides a cluster-level view of the `DEADLOCKS` table. | | [`CLUSTER_HARDWARE`](https://docs.pingcap.com/tidb/stable/information-schema-cluster-hardware) | Provides details on the underlying physical hardware discovered on each TiDB component. This table is not applicable to TiDB Cloud. | -| [`CLUSTER_INFO`](/information-schema/information-schema-cluster-info.md) | Provides details on the current cluster topology. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | +| [`CLUSTER_INFO`](/information-schema/information-schema-cluster-info.md) | Provides details on the current cluster topology. This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. | | [`CLUSTER_LOAD`](https://docs.pingcap.com/tidb/stable/information-schema-cluster-load) | Provides current load information for TiDB servers in the cluster. This table is not applicable to TiDB Cloud. | | [`CLUSTER_LOG`](https://docs.pingcap.com/tidb/stable/information-schema-cluster-log) | Provides a log for the entire TiDB cluster. This table is not applicable to TiDB Cloud. | | `CLUSTER_MEMORY_USAGE` | Provides a cluster-level view of the `MEMORY_USAGE` table. This table is not applicable to TiDB Cloud. | | `CLUSTER_MEMORY_USAGE_OPS_HISTORY` | Provides a cluster-level view of the `MEMORY_USAGE_OPS_HISTORY` table. This table is not applicable to TiDB Cloud. | | `CLUSTER_PROCESSLIST` | Provides a cluster-level view of the `PROCESSLIST` table. | -| `CLUSTER_SLOW_QUERY` | Provides a cluster-level view of the `SLOW_QUERY` table. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | -| `CLUSTER_STATEMENTS_SUMMARY` | Provides a cluster-level view of the `STATEMENTS_SUMMARY` table. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | -| `CLUSTER_STATEMENTS_SUMMARY_HISTORY` | Provides a cluster-level view of the `STATEMENTS_SUMMARY_HISTORY` table. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | +| `CLUSTER_SLOW_QUERY` | Provides a cluster-level view of the `SLOW_QUERY` table. This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. | +| `CLUSTER_STATEMENTS_SUMMARY` | Provides a cluster-level view of the `STATEMENTS_SUMMARY` table. This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. | +| `CLUSTER_STATEMENTS_SUMMARY_HISTORY` | Provides a cluster-level view of the `STATEMENTS_SUMMARY_HISTORY` table. This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. | | `CLUSTER_TIDB_TRX` | Provides a cluster-level view of the `TIDB_TRX` table. | | [`CLUSTER_SYSTEMINFO`](https://docs.pingcap.com/tidb/stable/information-schema-cluster-systeminfo) | Provides details about kernel parameter configuration for servers in the cluster. This table is not applicable to TiDB Cloud. | | [`DATA_LOCK_WAITS`](/information-schema/information-schema-data-lock-waits.md) | Provides the lock-waiting information on the TiKV server. | @@ -186,11 +188,11 @@ Many `INFORMATION_SCHEMA` tables have a corresponding `SHOW` command. The benefi | [`METRICS_SUMMARY`](https://docs.pingcap.com/tidb/stable/information-schema-metrics-summary) | A summary of metrics extracted from Prometheus. This table is not applicable to TiDB Cloud. | | `METRICS_SUMMARY_BY_LABEL` | See `METRICS_SUMMARY` table. This table is not applicable to TiDB Cloud. | | [`METRICS_TABLES`](https://docs.pingcap.com/tidb/stable/information-schema-metrics-tables) | Provides the PromQL definitions for tables in `METRICS_SCHEMA`. This table is not applicable to TiDB Cloud. | -| [`PLACEMENT_POLICIES`](https://docs.pingcap.com/tidb/stable/information-schema-placement-policies) | Provides information on all placement policies. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | +| [`PLACEMENT_POLICIES`](https://docs.pingcap.com/tidb/stable/information-schema-placement-policies) | Provides information on all placement policies. This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. | | [`SEQUENCES`](/information-schema/information-schema-sequences.md) | The TiDB implementation of sequences is based on MariaDB. | -| [`SLOW_QUERY`](/information-schema/information-schema-slow-query.md) | Provides information on slow queries on the current TiDB server. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | -| [`STATEMENTS_SUMMARY`](/statement-summary-tables.md) | Similar to performance_schema statement summary in MySQL. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | -| [`STATEMENTS_SUMMARY_HISTORY`](/statement-summary-tables.md) | Similar to performance_schema statement summary history in MySQL. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters.| +| [`SLOW_QUERY`](/information-schema/information-schema-slow-query.md) | Provides information on slow queries on the current TiDB server. This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. | +| [`STATEMENTS_SUMMARY`](/statement-summary-tables.md) | Similar to performance_schema statement summary in MySQL. This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. | +| [`STATEMENTS_SUMMARY_HISTORY`](/statement-summary-tables.md) | Similar to performance_schema statement summary history in MySQL. This table is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters.| | [`TABLE_STORAGE_STATS`](/information-schema/information-schema-table-storage-stats.md) | Provides details about table sizes in storage. | | [`TIDB_HOT_REGIONS`](https://docs.pingcap.com/tidb/stable/information-schema-tidb-hot-regions) | Provides statistics about which regions are hot. This table is not applicable to TiDB Cloud. | | [`TIDB_HOT_REGIONS_HISTORY`](/information-schema/information-schema-tidb-hot-regions-history.md) | Provides history statistics about which Regions are hot. | diff --git a/join-reorder.md b/join-reorder.md index 747c18365e058..295da0046927c 100644 --- a/join-reorder.md +++ b/join-reorder.md @@ -1,7 +1,6 @@ --- title: Introduction to Join Reorder summary: Use the Join Reorder algorithm to join multiple tables in TiDB. -aliases: ['/docs/dev/join-reorder/','/docs/dev/reference/performance/join-reorder/'] --- # Introduction to Join Reorder diff --git a/keywords.md b/keywords.md index 14a6ac92f2dc5..017a215a907a5 100644 --- a/keywords.md +++ b/keywords.md @@ -1,14 +1,13 @@ --- title: Keywords summary: Keywords and Reserved Words -aliases: ['/docs/dev/keywords-and-reserved-words/','/docs/dev/reference/sql/language-structure/keywords-and-reserved-words/','/tidb/dev/keywords-and-reserved-words/'] --- # Keywords This article introduces the keywords in TiDB, the differences between reserved words and non-reserved words and summarizes all keywords for the query. -Keywords are words that have special meanings in SQL statements, such as `SELECT`, `UPDATE`, and `DELETE`. Some of them can be used as identifiers directly, which are called **non-reserved keywords**. Some of them require special treatment before being used as identifiers, which are called **reserved keywords**. However, there are special non-reserved keywords that might still require special treatment. It is recommended that you treat them as reserved keywords. +Keywords are words that have special meanings in SQL statements, such as [`SELECT`](/sql-statements/sql-statement-select.md), [`UPDATE`](/sql-statements/sql-statement-update.md), and [`DELETE`](/sql-statements/sql-statement-delete.md). Some of them can be used as identifiers directly, which are called **non-reserved keywords**. Some of them require special treatment before being used as identifiers, which are called **reserved keywords**. To use the reserved keywords as identifiers, you must enclose them in backticks `` ` ``: @@ -56,9 +55,11 @@ CREATE TABLE test.select (BEGIN int, END int); Query OK, 0 rows affected (0.08 sec) ``` +Starting from v7.5.3, TiDB provides a full list of keywords in the [`INFORMATION_SCHEMA.KEYWORDS`](/information-schema/information-schema-keywords.md) table. + ## Keyword list -The following list shows the keywords in TiDB. Reserved keywords are marked with `(R)`. Reserved keywords for [Window Functions](/functions-and-operators/window-functions.md) are marked with `(R-Window)`. Special non-reserved keywords that need to be escaped with backticks `` ` `` are marked with `(S)`. +The following list shows the keywords in TiDB. Reserved keywords are marked with `(R)`. Reserved keywords for [Window Functions](/functions-and-operators/window-functions.md) are marked with `(R-Window)`. @@ -67,7 +68,7 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - ACCOUNT - ACTION - ADD (R) -- ADMIN (R) +- ADMIN - ADVISE - AFTER - AGAINST @@ -97,6 +98,8 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - BACKEND - BACKUP - BACKUPS +- BATCH +- BDR - BEGIN - BERNOULLI - BETWEEN (R) @@ -113,8 +116,8 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - BOOLEAN - BOTH (R) - BTREE -- BUCKETS (R) -- BUILTINS (R) +- BUCKETS +- BUILTINS - BY (R) - BYTE @@ -123,8 +126,9 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - CACHE - CALIBRATE - CALL (R) -- CANCEL (R) +- CANCEL - CAPTURE +- CARDINALITY - CASCADE (R) - CASCADED - CASE (R) @@ -144,12 +148,13 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - CLOSE - CLUSTER - CLUSTERED -- CMSKETCH (R) +- CMSKETCH - COALESCE - COLLATE (R) - COLLATION - COLUMN (R) - COLUMN_FORMAT +- COLUMN_STATS_USAGE - COLUMNS - COMMENT - COMMIT @@ -166,6 +171,7 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - CONTEXT - CONTINUE (R) - CONVERT (R) +- CORRELATION - CPU - CREATE (R) - CROSS (R) @@ -198,7 +204,7 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - DAY_MICROSECOND (R) - DAY_MINUTE (R) - DAY_SECOND (R) -- DDL (R) +- DDL - DEALLOCATE - DECIMAL (R) - DECLARE @@ -208,7 +214,8 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - DELAYED (R) - DELETE (R) - DENSE_RANK (R-Window) -- DEPTH (R) +- DEPENDENCY +- DEPTH - DESC (R) - DESCRIBE (R) - DIGEST @@ -222,8 +229,9 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - DIV (R) - DO - DOUBLE (R) -- DRAINER (R) +- DRAINER - DROP (R) +- DRY - DUAL (R) - DUPLICATE - DYNAMIC @@ -303,6 +311,7 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - HELP - HIGH_PRIORITY (R) - HISTOGRAM +- HISTOGRAMS_IN_FLIGHT - HISTORY - HOSTS - HOUR @@ -351,8 +360,8 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with J -- JOB (R) -- JOBS (R) +- JOB +- JOBS - JOIN (R) - JSON @@ -385,6 +394,7 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - LIST - LOAD (R) - LOCAL +- LOCAL_ONLY - LOCALTIME (R) - LOCALTIMESTAMP (R) - LOCATION @@ -440,8 +450,8 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - NOCACHE - NOCYCLE - NODEGROUP -- NODE_ID (R) -- NODE_STATE (R) +- NODE_ID +- NODE_STATE - NOMAXVALUE - NOMINVALUE - NONCLUSTERED @@ -469,7 +479,7 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - ONLINE - ONLY - OPEN -- OPTIMISTIC (R) +- OPTIMISTIC - OPTIMIZE (R) - OPTION (R) - OPTIONAL @@ -497,8 +507,7 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - PERCENT_RANK (R-Window) - PER_DB - PER_TABLE -- PESSIMISTIC (R) -- PLACEMENT (S) +- PESSIMISTIC - PLUGINS - POINT - POLICY @@ -515,7 +524,7 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - PROFILE - PROFILES - PROXY -- PUMP (R) +- PUMP - PURGE Q @@ -538,8 +547,8 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - REDUNDANT - REFERENCES (R) - REGEXP (R) -- REGION (R) -- REGIONS (R) +- REGION +- REGIONS - RELEASE (R) - RELOAD - REMOVE @@ -554,6 +563,7 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - REPLICATION - REQUIRE (R) - REQUIRED +- RESET - RESOURCE - RESPECT - RESTART @@ -576,14 +586,17 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - ROW_NUMBER (R-Window) - ROWS (R-Window) - RTREE +- RUN S -- SAMPLES (R) +- SAMPLERATE +- SAMPLES - SAN - SAVEPOINT - SECOND - SECOND_MICROSECOND (R) +- SECONDARY - SECONDARY_ENGINE - SECONDARY_LOAD - SECONDARY_UNLOAD @@ -595,6 +608,7 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - SERIAL - SERIALIZABLE - SESSION +- SESSION_STATES - SET (R) - SETVAL - SHARD_ROW_ID_BITS @@ -613,7 +627,7 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - SOME - SOURCE - SPATIAL (R) -- SPLIT (R) +- SPLIT - SQL (R) - SQL_BIG_RESULT (R) - SQL_BUFFER_RESULT @@ -635,19 +649,22 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - SSL (R) - START - STARTING (R) -- STATS (R) +- STATISTICS +- STATS - STATS_AUTO_RECALC -- STATS_BUCKETS (R) +- STATS_BUCKETS - STATS_COL_CHOICE - STATS_COL_LIST - STATS_EXTENDED (R) -- STATS_HEALTHY (R) -- STATS_HISTOGRAMS (R) -- STATS_META (R) +- STATS_HEALTHY +- STATS_HISTOGRAMS +- STATS_LOCKED +- STATS_META - STATS_OPTIONS - STATS_PERSISTENT - STATS_SAMPLE_PAGES - STATS_SAMPLE_RATE +- STATS_TOPN - STATUS - STORAGE - STORED (R) @@ -669,15 +686,17 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - TABLESAMPLE (R) - TABLESPACE - TABLE_CHECKSUM +- TELEMETRY +- TELEMETRY_ID - TEMPORARY - TEMPTABLE - TERMINATED (R) - TEXT - THAN - THEN (R) -- TIDB (R) +- TIDB - TiDB_CURRENT_TSO (R) -- TIFLASH (R) +- TIFLASH - TIKV_IMPORTER - TIME - TIMESTAMP @@ -686,7 +705,7 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - TINYTEXT (R) - TO (R) - TOKEN_ISSUER -- TOPN (R) +- TOPN - TPCC - TPCH_10 - TRACE @@ -697,6 +716,7 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - TRIGGERS - TRUE (R) - TRUNCATE +- TSO - TTL - TTL_ENABLE - TTL_JOB_INTERVAL @@ -746,7 +766,7 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - WHEN (R) - WHERE (R) - WHILE (R) -- WIDTH (R) +- WIDTH - WINDOW (R-Window) - WITH (R) - WITHOUT diff --git a/literal-values.md b/literal-values.md index f0048492460a6..f0de55559710a 100644 --- a/literal-values.md +++ b/literal-values.md @@ -1,7 +1,6 @@ --- title: Literal Values summary: This article introduces the literal values ​​of TiDB SQL statements. -aliases: ['/docs/dev/literal-values/','/docs/dev/reference/sql/language-structure/literal-values/'] --- # Literal Values diff --git a/maintain-tidb-using-tiup.md b/maintain-tidb-using-tiup.md index 18e30b89617fe..d853443f62a8a 100644 --- a/maintain-tidb-using-tiup.md +++ b/maintain-tidb-using-tiup.md @@ -1,7 +1,6 @@ --- title: TiUP Common Operations summary: Learn the common operations to operate and maintain a TiDB cluster using TiUP. -aliases: ['/docs/dev/maintain-tidb-using-tiup/','/docs/dev/how-to/maintain/tiup-operations/'] --- # TiUP Common Operations @@ -113,7 +112,7 @@ When the cluster is in operation, if you need to modify the parameters of a comp **Use `.` to represent the hierarchy of the configuration items**. - For more information on the configuration parameters of components, refer to [TiDB `config.toml.example`](https://github.com/pingcap/tidb/blob/master/pkg/config/config.toml.example), [TiKV `config.toml.example`](https://github.com/tikv/tikv/blob/master/etc/config-template.toml), and [PD `config.toml.example`](https://github.com/tikv/pd/blob/master/conf/config.toml). + For more information on the configuration parameters of components, refer to [TiDB `config.toml.example`](https://github.com/pingcap/tidb/blob/release-7.5/pkg/config/config.toml.example), [TiKV `config.toml.example`](https://github.com/tikv/tikv/blob/release-7.5/etc/config-template.toml), and [PD `config.toml.example`](https://github.com/tikv/pd/blob/release-7.5/conf/config.toml). 3. Rolling update the configuration and restart the corresponding components by running the `reload` command: @@ -125,7 +124,7 @@ When the cluster is in operation, if you need to modify the parameters of a comp ### Example -If you want to set the transaction size limit parameter (`txn-total-size-limit` in the [performance](https://github.com/pingcap/tidb/blob/master/pkg/config/config.toml.example) module) to `1G` in tidb-server, edit the configuration as follows: +If you want to set the transaction size limit parameter (`txn-total-size-limit` in the [performance](https://github.com/pingcap/tidb/blob/release-7.5/pkg/config/config.toml.example) module) to `1G` in tidb-server, edit the configuration as follows: ``` server_configs: diff --git a/media/best-practices/commit-log-duration.png b/media/best-practices/commit-log-duration.png new file mode 100644 index 0000000000000..de889199aeb62 Binary files /dev/null and b/media/best-practices/commit-log-duration.png differ diff --git a/media/dashboard/dashboard-statement-detail0.png b/media/dashboard/dashboard-statement-detail0.png new file mode 100644 index 0000000000000..b066c5e3c4a88 Binary files /dev/null and b/media/dashboard/dashboard-statement-detail0.png differ diff --git a/media/dashboard/dashboard-statement-detail1.png b/media/dashboard/dashboard-statement-detail1.png new file mode 100644 index 0000000000000..b17cb3797711f Binary files /dev/null and b/media/dashboard/dashboard-statement-detail1.png differ diff --git a/media/dashboard/dashboard-statement-detail2.png b/media/dashboard/dashboard-statement-detail2.png new file mode 100644 index 0000000000000..8cfedf64b1c0c Binary files /dev/null and b/media/dashboard/dashboard-statement-detail2.png differ diff --git a/media/develop/autocommit_selectforupdate_nowaitlock.png b/media/develop/autocommit_selectforupdate_nowaitlock.png index 3f495ff79afda..f2d0563d58150 100644 Binary files a/media/develop/autocommit_selectforupdate_nowaitlock.png and b/media/develop/autocommit_selectforupdate_nowaitlock.png differ diff --git a/media/develop/mysql-workbench-add-new-connection.png b/media/develop/mysql-workbench-add-new-connection.png new file mode 100644 index 0000000000000..5e4426063d1bd Binary files /dev/null and b/media/develop/mysql-workbench-add-new-connection.png differ diff --git a/media/develop/mysql-workbench-adjust-sqleditor-read-timeout.jpg b/media/develop/mysql-workbench-adjust-sqleditor-read-timeout.jpg new file mode 100644 index 0000000000000..0891f2741f8e3 Binary files /dev/null and b/media/develop/mysql-workbench-adjust-sqleditor-read-timeout.jpg differ diff --git a/media/develop/mysql-workbench-connection-config-dedicated-parameters.png b/media/develop/mysql-workbench-connection-config-dedicated-parameters.png new file mode 100644 index 0000000000000..9b303c5c15d7d Binary files /dev/null and b/media/develop/mysql-workbench-connection-config-dedicated-parameters.png differ diff --git a/media/develop/mysql-workbench-connection-config-self-hosted-parameters.png b/media/develop/mysql-workbench-connection-config-self-hosted-parameters.png new file mode 100644 index 0000000000000..edf9e14cefc37 Binary files /dev/null and b/media/develop/mysql-workbench-connection-config-self-hosted-parameters.png differ diff --git a/media/develop/mysql-workbench-connection-config-serverless-parameters.png b/media/develop/mysql-workbench-connection-config-serverless-parameters.png new file mode 100644 index 0000000000000..81bce73f47f1e Binary files /dev/null and b/media/develop/mysql-workbench-connection-config-serverless-parameters.png differ diff --git a/media/develop/mysql-workbench-store-dedicated-password-in-keychain.png b/media/develop/mysql-workbench-store-dedicated-password-in-keychain.png new file mode 100644 index 0000000000000..60359cae14c83 Binary files /dev/null and b/media/develop/mysql-workbench-store-dedicated-password-in-keychain.png differ diff --git a/media/develop/mysql-workbench-store-password-in-keychain.png b/media/develop/mysql-workbench-store-password-in-keychain.png new file mode 100644 index 0000000000000..b65f4bcd622b8 Binary files /dev/null and b/media/develop/mysql-workbench-store-password-in-keychain.png differ diff --git a/media/develop/mysql-workbench-store-self-hosted-password-in-keychain.png b/media/develop/mysql-workbench-store-self-hosted-password-in-keychain.png new file mode 100644 index 0000000000000..8f7f66e62e2f2 Binary files /dev/null and b/media/develop/mysql-workbench-store-self-hosted-password-in-keychain.png differ diff --git a/media/develop/navicat-add-new-connection.jpg b/media/develop/navicat-add-new-connection.jpg new file mode 100644 index 0000000000000..13662f4174066 Binary files /dev/null and b/media/develop/navicat-add-new-connection.jpg differ diff --git a/media/develop/navicat-connection-config-dedicated-general.png b/media/develop/navicat-connection-config-dedicated-general.png new file mode 100644 index 0000000000000..8d41a772cf045 Binary files /dev/null and b/media/develop/navicat-connection-config-dedicated-general.png differ diff --git a/media/develop/navicat-connection-config-dedicated-ssl.jpg b/media/develop/navicat-connection-config-dedicated-ssl.jpg new file mode 100644 index 0000000000000..3b5efb0438225 Binary files /dev/null and b/media/develop/navicat-connection-config-dedicated-ssl.jpg differ diff --git a/media/develop/navicat-connection-config-self-hosted-general.png b/media/develop/navicat-connection-config-self-hosted-general.png new file mode 100644 index 0000000000000..058e7789d824a Binary files /dev/null and b/media/develop/navicat-connection-config-self-hosted-general.png differ diff --git a/media/develop/navicat-connection-config-serverless-general.png b/media/develop/navicat-connection-config-serverless-general.png new file mode 100644 index 0000000000000..a1e061c018a38 Binary files /dev/null and b/media/develop/navicat-connection-config-serverless-general.png differ diff --git a/media/develop/navicat-connection-config-serverless-ssl.png b/media/develop/navicat-connection-config-serverless-ssl.png new file mode 100644 index 0000000000000..bb2bcbce45e97 Binary files /dev/null and b/media/develop/navicat-connection-config-serverless-ssl.png differ diff --git a/media/develop/navicat-premium-add-new-connection.png b/media/develop/navicat-premium-add-new-connection.png new file mode 100644 index 0000000000000..f91f1f495719b Binary files /dev/null and b/media/develop/navicat-premium-add-new-connection.png differ diff --git a/media/develop/navicat-premium-connection-config-dedicated-general.png b/media/develop/navicat-premium-connection-config-dedicated-general.png new file mode 100644 index 0000000000000..2dfb34692d661 Binary files /dev/null and b/media/develop/navicat-premium-connection-config-dedicated-general.png differ diff --git a/media/develop/navicat-premium-connection-config-dedicated-ssl.png b/media/develop/navicat-premium-connection-config-dedicated-ssl.png new file mode 100644 index 0000000000000..5ea365072fe51 Binary files /dev/null and b/media/develop/navicat-premium-connection-config-dedicated-ssl.png differ diff --git a/media/develop/navicat-premium-connection-config-self-hosted-general.png b/media/develop/navicat-premium-connection-config-self-hosted-general.png new file mode 100644 index 0000000000000..5e3cd7df34074 Binary files /dev/null and b/media/develop/navicat-premium-connection-config-self-hosted-general.png differ diff --git a/media/develop/navicat-premium-connection-config-serverless-general.png b/media/develop/navicat-premium-connection-config-serverless-general.png new file mode 100644 index 0000000000000..d8b6ea1f5854b Binary files /dev/null and b/media/develop/navicat-premium-connection-config-serverless-general.png differ diff --git a/media/develop/navicat-premium-connection-config-serverless-ssl.png b/media/develop/navicat-premium-connection-config-serverless-ssl.png new file mode 100644 index 0000000000000..7bdb79e16fc93 Binary files /dev/null and b/media/develop/navicat-premium-connection-config-serverless-ssl.png differ diff --git a/media/grafana-password-reset1.png b/media/grafana-password-reset1.png new file mode 100644 index 0000000000000..b8559c659cf7c Binary files /dev/null and b/media/grafana-password-reset1.png differ diff --git a/media/grafana-password-reset2.png b/media/grafana-password-reset2.png new file mode 100644 index 0000000000000..0c7c604d0fda6 Binary files /dev/null and b/media/grafana-password-reset2.png differ diff --git a/media/region-panel.png b/media/region-panel.png index 5b1fa4520c5ea..578a59cd87ef6 100644 Binary files a/media/region-panel.png and b/media/region-panel.png differ diff --git a/media/ticdc/ticdc-changefeed-state-transfer.png b/media/ticdc/ticdc-changefeed-state-transfer.png index 08802299c1cce..c0e1fb84feccc 100644 Binary files a/media/ticdc/ticdc-changefeed-state-transfer.png and b/media/ticdc/ticdc-changefeed-state-transfer.png differ diff --git a/media/tidb-cloud/Project-CIDR2.png b/media/tidb-cloud/Project-CIDR2.png index b412a4889f229..b0739027bda67 100644 Binary files a/media/tidb-cloud/Project-CIDR2.png and b/media/tidb-cloud/Project-CIDR2.png differ diff --git a/media/tidb-cloud/Project-CIDR4.png b/media/tidb-cloud/Project-CIDR4.png index 0879853b29d08..e73e2b3bb87a5 100644 Binary files a/media/tidb-cloud/Project-CIDR4.png and b/media/tidb-cloud/Project-CIDR4.png differ diff --git a/media/tidb-cloud/VPC-Peering3.png b/media/tidb-cloud/VPC-Peering3.png deleted file mode 100644 index b3a8aacbb72c3..0000000000000 Binary files a/media/tidb-cloud/VPC-Peering3.png and /dev/null differ diff --git a/media/tidb-cloud/aws-dms-from-oracle-to-tidb-0.png b/media/tidb-cloud/aws-dms-from-oracle-to-tidb-0.png index 483dcb385cd40..09e9b3cb8cafb 100644 Binary files a/media/tidb-cloud/aws-dms-from-oracle-to-tidb-0.png and b/media/tidb-cloud/aws-dms-from-oracle-to-tidb-0.png differ diff --git a/media/tidb-cloud/aws-dms-tidb-cloud/aws-dms-connect-configure-endpoint.png b/media/tidb-cloud/aws-dms-tidb-cloud/aws-dms-connect-configure-endpoint.png new file mode 100644 index 0000000000000..66453ddce0da5 Binary files /dev/null and b/media/tidb-cloud/aws-dms-tidb-cloud/aws-dms-connect-configure-endpoint.png differ diff --git a/media/tidb-cloud/aws-dms-tidb-cloud/aws-dms-connect-connectivity-security.png b/media/tidb-cloud/aws-dms-tidb-cloud/aws-dms-connect-connectivity-security.png new file mode 100644 index 0000000000000..2b8fbd872d355 Binary files /dev/null and b/media/tidb-cloud/aws-dms-tidb-cloud/aws-dms-connect-connectivity-security.png differ diff --git a/media/tidb-cloud/aws-dms-tidb-cloud/aws-dms-connect-create-endpoint.png b/media/tidb-cloud/aws-dms-tidb-cloud/aws-dms-connect-create-endpoint.png new file mode 100644 index 0000000000000..bdf6d097ff92f Binary files /dev/null and b/media/tidb-cloud/aws-dms-tidb-cloud/aws-dms-connect-create-endpoint.png differ diff --git a/media/tidb-cloud/aws-dms-tidb-cloud/aws-dms-connect-replication-instances.png b/media/tidb-cloud/aws-dms-tidb-cloud/aws-dms-connect-replication-instances.png new file mode 100644 index 0000000000000..56aebc9d9eb6b Binary files /dev/null and b/media/tidb-cloud/aws-dms-tidb-cloud/aws-dms-connect-replication-instances.png differ diff --git a/media/tidb-cloud/changefeed/connect-to-aws-self-hosted-kafka-privatelink-service.jpeg b/media/tidb-cloud/changefeed/connect-to-aws-self-hosted-kafka-privatelink-service.jpeg new file mode 100644 index 0000000000000..d0d5fa35a0555 Binary files /dev/null and b/media/tidb-cloud/changefeed/connect-to-aws-self-hosted-kafka-privatelink-service.jpeg differ diff --git a/media/tidb-cloud/changefeed/connect-to-google-cloud-self-hosted-kafka-private-service-connect-by-kafka-proxy.jpeg b/media/tidb-cloud/changefeed/connect-to-google-cloud-self-hosted-kafka-private-service-connect-by-kafka-proxy.jpeg new file mode 100644 index 0000000000000..290ee89bde5e3 Binary files /dev/null and b/media/tidb-cloud/changefeed/connect-to-google-cloud-self-hosted-kafka-private-service-connect-by-kafka-proxy.jpeg differ diff --git a/media/tidb-cloud/changefeed/connect-to-google-cloud-self-hosted-kafka-private-service-connect-by-portmapping.jpeg b/media/tidb-cloud/changefeed/connect-to-google-cloud-self-hosted-kafka-private-service-connect-by-portmapping.jpeg new file mode 100644 index 0000000000000..d0b3595663820 Binary files /dev/null and b/media/tidb-cloud/changefeed/connect-to-google-cloud-self-hosted-kafka-private-service-connect-by-portmapping.jpeg differ diff --git a/media/tidb-cloud/data-service/GPTs1.png b/media/tidb-cloud/data-service/GPTs1.png new file mode 100644 index 0000000000000..b9b5d51ebf44d Binary files /dev/null and b/media/tidb-cloud/data-service/GPTs1.png differ diff --git a/media/tidb-cloud/data-service/GPTs2.png b/media/tidb-cloud/data-service/GPTs2.png new file mode 100644 index 0000000000000..82f0fb37350ca Binary files /dev/null and b/media/tidb-cloud/data-service/GPTs2.png differ diff --git a/media/tidb-cloud/dm-billing-cross-az-fees.png b/media/tidb-cloud/dm-billing-cross-az-fees.png index 614a647971e6a..3131df528cf94 100644 Binary files a/media/tidb-cloud/dm-billing-cross-az-fees.png and b/media/tidb-cloud/dm-billing-cross-az-fees.png differ diff --git a/media/tidb-cloud/dm-billing-cross-region-and-az-fees.png b/media/tidb-cloud/dm-billing-cross-region-and-az-fees.png index 0c6b65d0bed3b..3226655ac9545 100644 Binary files a/media/tidb-cloud/dm-billing-cross-region-and-az-fees.png and b/media/tidb-cloud/dm-billing-cross-region-and-az-fees.png differ diff --git a/media/tidb-cloud/dm-billing-cross-region-fees.png b/media/tidb-cloud/dm-billing-cross-region-fees.png index adbef21484419..dd267f6cb3b72 100644 Binary files a/media/tidb-cloud/dm-billing-cross-region-fees.png and b/media/tidb-cloud/dm-billing-cross-region-fees.png differ diff --git a/media/tidb-cloud/import-data-csv-config.png b/media/tidb-cloud/import-data-csv-config.png index bf469faa0112f..0f769a5d3aa17 100644 Binary files a/media/tidb-cloud/import-data-csv-config.png and b/media/tidb-cloud/import-data-csv-config.png differ diff --git a/media/tidb-cloud/looker-studio-configure-connection.png b/media/tidb-cloud/looker-studio-configure-connection.png new file mode 100644 index 0000000000000..82f5055f897fa Binary files /dev/null and b/media/tidb-cloud/looker-studio-configure-connection.png differ diff --git a/media/tidb-cloud/looker-studio-custom-query.png b/media/tidb-cloud/looker-studio-custom-query.png new file mode 100644 index 0000000000000..8426e2da3c71a Binary files /dev/null and b/media/tidb-cloud/looker-studio-custom-query.png differ diff --git a/media/tidb-cloud/looker-studio-simple-chart.png b/media/tidb-cloud/looker-studio-simple-chart.png new file mode 100644 index 0000000000000..257f9b19203da Binary files /dev/null and b/media/tidb-cloud/looker-studio-simple-chart.png differ diff --git a/media/tidb-cloud/recovery-group/recovery-group-failover.png b/media/tidb-cloud/recovery-group/recovery-group-failover.png new file mode 100644 index 0000000000000..ff2c15c37a520 Binary files /dev/null and b/media/tidb-cloud/recovery-group/recovery-group-failover.png differ diff --git a/media/tidb-cloud/recovery-group/recovery-group-overview.png b/media/tidb-cloud/recovery-group/recovery-group-overview.png new file mode 100644 index 0000000000000..fe97b0067ce39 Binary files /dev/null and b/media/tidb-cloud/recovery-group/recovery-group-overview.png differ diff --git a/media/tidb-cloud/recovery-group/recovery-group-protected.png b/media/tidb-cloud/recovery-group/recovery-group-protected.png new file mode 100644 index 0000000000000..5e44702cb732e Binary files /dev/null and b/media/tidb-cloud/recovery-group/recovery-group-protected.png differ diff --git a/media/tidb-cloud/recovery-group/recovery-group-reprotected.png b/media/tidb-cloud/recovery-group/recovery-group-reprotected.png new file mode 100644 index 0000000000000..7d1afe53ed384 Binary files /dev/null and b/media/tidb-cloud/recovery-group/recovery-group-reprotected.png differ diff --git a/media/tidb-cloud/recovery-group/recovery-group-unprotected.png b/media/tidb-cloud/recovery-group/recovery-group-unprotected.png new file mode 100644 index 0000000000000..9597a81be24a5 Binary files /dev/null and b/media/tidb-cloud/recovery-group/recovery-group-unprotected.png differ diff --git a/media/tidb-cloud/serverless-external-storage/azure-sas-create.png b/media/tidb-cloud/serverless-external-storage/azure-sas-create.png new file mode 100644 index 0000000000000..f8c50e42359d9 Binary files /dev/null and b/media/tidb-cloud/serverless-external-storage/azure-sas-create.png differ diff --git a/media/tidb-cloud/serverless-external-storage/azure-sas-position.png b/media/tidb-cloud/serverless-external-storage/azure-sas-position.png new file mode 100644 index 0000000000000..01fbb2edaf7b0 Binary files /dev/null and b/media/tidb-cloud/serverless-external-storage/azure-sas-position.png differ diff --git a/media/tidb-cloud/serverless-external-storage/gcs-service-account-key.png b/media/tidb-cloud/serverless-external-storage/gcs-service-account-key.png new file mode 100644 index 0000000000000..655c502fbab4b Binary files /dev/null and b/media/tidb-cloud/serverless-external-storage/gcs-service-account-key.png differ diff --git a/media/tidb-cloud/serverless-external-storage/gcs-service-account.png b/media/tidb-cloud/serverless-external-storage/gcs-service-account.png new file mode 100644 index 0000000000000..7ab07dc7c8885 Binary files /dev/null and b/media/tidb-cloud/serverless-external-storage/gcs-service-account.png differ diff --git a/media/tidb-cloud/serverless-external-storage/serverless-role-arn.png b/media/tidb-cloud/serverless-external-storage/serverless-role-arn.png new file mode 100644 index 0000000000000..509f1bf06940d Binary files /dev/null and b/media/tidb-cloud/serverless-external-storage/serverless-role-arn.png differ diff --git a/media/tidb-cloud/serverless-regional-high-avaliability-aws.png b/media/tidb-cloud/serverless-regional-high-avaliability-aws.png new file mode 100644 index 0000000000000..368373b69bef2 Binary files /dev/null and b/media/tidb-cloud/serverless-regional-high-avaliability-aws.png differ diff --git a/media/tidb-cloud/serverless-zonal-high-avaliability-aws.png b/media/tidb-cloud/serverless-zonal-high-avaliability-aws.png new file mode 100644 index 0000000000000..2a2d5739640c6 Binary files /dev/null and b/media/tidb-cloud/serverless-zonal-high-avaliability-aws.png differ diff --git a/media/tidb-cloud/tidb-cloud-alert-subscription.png b/media/tidb-cloud/tidb-cloud-alert-subscription.png new file mode 100644 index 0000000000000..6dfaa96d269c5 Binary files /dev/null and b/media/tidb-cloud/tidb-cloud-alert-subscription.png differ diff --git a/media/tidb-cloud/tidb-cloud-clinic.png b/media/tidb-cloud/tidb-cloud-clinic.png new file mode 100644 index 0000000000000..6b7944062c0db Binary files /dev/null and b/media/tidb-cloud/tidb-cloud-clinic.png differ diff --git a/media/tidb-cloud/v.7.1.0-oltp_read_write.png b/media/tidb-cloud/v.7.1.0-oltp_read_write.png deleted file mode 100644 index 0b56a59afcac0..0000000000000 Binary files a/media/tidb-cloud/v.7.1.0-oltp_read_write.png and /dev/null differ diff --git a/media/tidb-cloud/v6.5.6-oltp_insert.png b/media/tidb-cloud/v6.5.6-oltp_insert.png new file mode 100644 index 0000000000000..0ee03639ce2bf Binary files /dev/null and b/media/tidb-cloud/v6.5.6-oltp_insert.png differ diff --git a/media/tidb-cloud/v6.5.6-oltp_read_write.png b/media/tidb-cloud/v6.5.6-oltp_read_write.png new file mode 100644 index 0000000000000..82e5ab9075603 Binary files /dev/null and b/media/tidb-cloud/v6.5.6-oltp_read_write.png differ diff --git a/media/tidb-cloud/v6.5.6-oltp_select_point.png b/media/tidb-cloud/v6.5.6-oltp_select_point.png new file mode 100644 index 0000000000000..19496cb0e42b0 Binary files /dev/null and b/media/tidb-cloud/v6.5.6-oltp_select_point.png differ diff --git a/media/tidb-cloud/v6.5.6-oltp_update_index.png b/media/tidb-cloud/v6.5.6-oltp_update_index.png new file mode 100644 index 0000000000000..9cd6ab54f7cc9 Binary files /dev/null and b/media/tidb-cloud/v6.5.6-oltp_update_index.png differ diff --git a/media/tidb-cloud/v6.5.6-oltp_update_non_index.png b/media/tidb-cloud/v6.5.6-oltp_update_non_index.png new file mode 100644 index 0000000000000..235529282bc2a Binary files /dev/null and b/media/tidb-cloud/v6.5.6-oltp_update_non_index.png differ diff --git a/media/tidb-cloud/v6.5.6-tpmC.png b/media/tidb-cloud/v6.5.6-tpmC.png new file mode 100644 index 0000000000000..d454c4492ed21 Binary files /dev/null and b/media/tidb-cloud/v6.5.6-tpmC.png differ diff --git a/media/tidb-cloud/v7.1.3-oltp_insert.png b/media/tidb-cloud/v7.1.3-oltp_insert.png new file mode 100644 index 0000000000000..b98251fbae658 Binary files /dev/null and b/media/tidb-cloud/v7.1.3-oltp_insert.png differ diff --git a/media/tidb-cloud/v7.1.3-oltp_read_write.png b/media/tidb-cloud/v7.1.3-oltp_read_write.png new file mode 100644 index 0000000000000..1d21b6573d51b Binary files /dev/null and b/media/tidb-cloud/v7.1.3-oltp_read_write.png differ diff --git a/media/tidb-cloud/v7.1.3-oltp_select_point.png b/media/tidb-cloud/v7.1.3-oltp_select_point.png new file mode 100644 index 0000000000000..5d619973bdcd0 Binary files /dev/null and b/media/tidb-cloud/v7.1.3-oltp_select_point.png differ diff --git a/media/tidb-cloud/v7.1.3-oltp_update_index.png b/media/tidb-cloud/v7.1.3-oltp_update_index.png new file mode 100644 index 0000000000000..2ed956c736dde Binary files /dev/null and b/media/tidb-cloud/v7.1.3-oltp_update_index.png differ diff --git a/media/tidb-cloud/v7.1.3-oltp_update_non_index.png b/media/tidb-cloud/v7.1.3-oltp_update_non_index.png new file mode 100644 index 0000000000000..74aa87acdecdd Binary files /dev/null and b/media/tidb-cloud/v7.1.3-oltp_update_non_index.png differ diff --git a/media/tidb-cloud/v7.1.3-tpmC.png b/media/tidb-cloud/v7.1.3-tpmC.png new file mode 100644 index 0000000000000..d8bf49e923c69 Binary files /dev/null and b/media/tidb-cloud/v7.1.3-tpmC.png differ diff --git a/media/tidb-cloud/v7.5.0-oltp_insert.png b/media/tidb-cloud/v7.5.0-oltp_insert.png new file mode 100644 index 0000000000000..00229755d4d29 Binary files /dev/null and b/media/tidb-cloud/v7.5.0-oltp_insert.png differ diff --git a/media/tidb-cloud/v7.5.0-oltp_point_select.png b/media/tidb-cloud/v7.5.0-oltp_point_select.png new file mode 100644 index 0000000000000..12c34021f206b Binary files /dev/null and b/media/tidb-cloud/v7.5.0-oltp_point_select.png differ diff --git a/media/tidb-cloud/v7.5.0-oltp_read_write.png b/media/tidb-cloud/v7.5.0-oltp_read_write.png new file mode 100644 index 0000000000000..b423451bd21ff Binary files /dev/null and b/media/tidb-cloud/v7.5.0-oltp_read_write.png differ diff --git a/media/tidb-cloud/v7.5.0-oltp_update_index.png b/media/tidb-cloud/v7.5.0-oltp_update_index.png new file mode 100644 index 0000000000000..ad3c303c886d0 Binary files /dev/null and b/media/tidb-cloud/v7.5.0-oltp_update_index.png differ diff --git a/media/tidb-cloud/v7.5.0-oltp_update_non_index.png b/media/tidb-cloud/v7.5.0-oltp_update_non_index.png new file mode 100644 index 0000000000000..ce97e5a4deb50 Binary files /dev/null and b/media/tidb-cloud/v7.5.0-oltp_update_non_index.png differ diff --git a/media/tidb-cloud/v7.5.0_tpcc.png b/media/tidb-cloud/v7.5.0_tpcc.png new file mode 100644 index 0000000000000..26089ecb4408c Binary files /dev/null and b/media/tidb-cloud/v7.5.0_tpcc.png differ diff --git a/media/tidb-cloud/v8.1.0_oltp_insert.png b/media/tidb-cloud/v8.1.0_oltp_insert.png new file mode 100644 index 0000000000000..fe0b422966171 Binary files /dev/null and b/media/tidb-cloud/v8.1.0_oltp_insert.png differ diff --git a/media/tidb-cloud/v8.1.0_oltp_point_select.png b/media/tidb-cloud/v8.1.0_oltp_point_select.png new file mode 100644 index 0000000000000..a1eb7e3029d76 Binary files /dev/null and b/media/tidb-cloud/v8.1.0_oltp_point_select.png differ diff --git a/media/tidb-cloud/v8.1.0_oltp_read_write.png b/media/tidb-cloud/v8.1.0_oltp_read_write.png new file mode 100644 index 0000000000000..75e9278d1f290 Binary files /dev/null and b/media/tidb-cloud/v8.1.0_oltp_read_write.png differ diff --git a/media/tidb-cloud/v8.1.0_oltp_update_index.png b/media/tidb-cloud/v8.1.0_oltp_update_index.png new file mode 100644 index 0000000000000..b5d58fe531600 Binary files /dev/null and b/media/tidb-cloud/v8.1.0_oltp_update_index.png differ diff --git a/media/tidb-cloud/v8.1.0_oltp_update_non_index.png b/media/tidb-cloud/v8.1.0_oltp_update_non_index.png new file mode 100644 index 0000000000000..b8e760933506d Binary files /dev/null and b/media/tidb-cloud/v8.1.0_oltp_update_non_index.png differ diff --git a/media/tidb-cloud/v8.1.0_tpcc.png b/media/tidb-cloud/v8.1.0_tpcc.png new file mode 100644 index 0000000000000..10eb8ef603758 Binary files /dev/null and b/media/tidb-cloud/v8.1.0_tpcc.png differ diff --git a/media/tidb-cloud/vercel/integration-link-cluster-page.png b/media/tidb-cloud/vercel/integration-link-cluster-page.png index 511d2e71b5f74..d5868ad9e6e3c 100644 Binary files a/media/tidb-cloud/vercel/integration-link-cluster-page.png and b/media/tidb-cloud/vercel/integration-link-cluster-page.png differ diff --git a/media/tidb-cloud/vercel/integration-vercel-configuration-page.png b/media/tidb-cloud/vercel/integration-vercel-configuration-page.png index 8ded906064965..f02a7f5d259fe 100644 Binary files a/media/tidb-cloud/vercel/integration-vercel-configuration-page.png and b/media/tidb-cloud/vercel/integration-vercel-configuration-page.png differ diff --git a/media/tidb-cloud/vercel/preview-envs.png b/media/tidb-cloud/vercel/preview-envs.png new file mode 100644 index 0000000000000..c37b53ccdfdf8 Binary files /dev/null and b/media/tidb-cloud/vercel/preview-envs.png differ diff --git a/media/tidb-cloud/vercel/tidbcloud-branch-check.png b/media/tidb-cloud/vercel/tidbcloud-branch-check.png new file mode 100644 index 0000000000000..34a3a33e8b0a4 Binary files /dev/null and b/media/tidb-cloud/vercel/tidbcloud-branch-check.png differ diff --git a/media/tidb-cloud/vercel/vercel-preview-deployment.png b/media/tidb-cloud/vercel/vercel-preview-deployment.png new file mode 100644 index 0000000000000..bf50ba2a02b31 Binary files /dev/null and b/media/tidb-cloud/vercel/vercel-preview-deployment.png differ diff --git a/media/tiflash/tiflash_mintso_v1.png b/media/tiflash/tiflash_mintso_v1.png new file mode 100644 index 0000000000000..964dcc6daacfe Binary files /dev/null and b/media/tiflash/tiflash_mintso_v1.png differ diff --git a/media/tiflash/tiflash_mintso_v2.png b/media/tiflash/tiflash_mintso_v2.png new file mode 100644 index 0000000000000..28d49cf7ab866 Binary files /dev/null and b/media/tiflash/tiflash_mintso_v2.png differ diff --git a/media/tikv-dashboard-flow-control.png b/media/tikv-dashboard-flow-control.png new file mode 100644 index 0000000000000..371c485ad82c1 Binary files /dev/null and b/media/tikv-dashboard-flow-control.png differ diff --git a/media/vector-search/embedding-search.png b/media/vector-search/embedding-search.png new file mode 100644 index 0000000000000..0035ada1b7479 Binary files /dev/null and b/media/vector-search/embedding-search.png differ diff --git a/metadata-lock.md b/metadata-lock.md index fbf663a184813..c16175a227a82 100644 --- a/metadata-lock.md +++ b/metadata-lock.md @@ -64,7 +64,7 @@ Starting from v6.5.0, TiDB enables metadata lock by default. When you upgrade yo | `COMMIT;` | | | `BEGIN;` | | | | `ALTER TABLE t MODIFY COLUMN a CHAR(10);` | - | `SELECT * FROM t;` (returns `Error 8028: Information schema is changed`) | | + | `SELECT * FROM t;` (returns `ERROR 8028 (HY000): public column a has changed`) | | ## Observability diff --git a/metrics-schema.md b/metrics-schema.md index 8462745adb995..3a8b19fdf892c 100644 --- a/metrics-schema.md +++ b/metrics-schema.md @@ -1,7 +1,6 @@ --- title: Metrics Schema summary: Learn the `METRICS_SCHEMA` schema. -aliases: ['/docs/dev/system-tables/system-table-metrics-schema/','/docs/dev/reference/system-databases/metrics-schema/','/tidb/dev/system-table-metrics-schema/'] --- # Metrics Schema diff --git a/migrate-aurora-to-tidb.md b/migrate-aurora-to-tidb.md index cb6de953e2d7b..12cbc69b97a58 100644 --- a/migrate-aurora-to-tidb.md +++ b/migrate-aurora-to-tidb.md @@ -1,7 +1,6 @@ --- title: Migrate Data from Amazon Aurora to TiDB summary: Learn how to migrate data from Amazon Aurora to TiDB using DB snapshot. -aliases: ['/tidb/dev/migrate-from-aurora-using-lightning','/docs/dev/migrate-from-aurora-mysql-database/','/docs/dev/how-to/migrate/from-mysql-aurora/','/docs/dev/how-to/migrate/from-aurora/', '/tidb/dev/migrate-from-aurora-mysql-database/', '/tidb/dev/migrate-from-mysql-aurora/','/tidb/stable/migrate-from-aurora-using-lightning/'] --- # Migrate Data from Amazon Aurora to TiDB @@ -51,7 +50,7 @@ Create a new `tidb-lightning-schema.toml` file, copy the following content into # The target TiDB cluster information. host = ${host} port = ${port} -user = "${user_name} +user = "${user_name}" password = "${password}" status-port = ${status-port} # The TiDB status port. Usually the port is 10080. pd-addr = "${ip}:${port}" # The cluster PD address. Usually the port is 2379. @@ -120,7 +119,7 @@ Create a new `tidb-lightning-data.toml` configuration file, copy the following c # The target TiDB cluster information. host = ${host} port = ${port} -user = "${user_name} +user = "${user_name}" password = "${password}" status-port = ${status-port} # The TiDB status port. Usually the port is 10080. pd-addr = "${ip}:${port}" # The cluster PD address. Usually the port is 2379. diff --git a/migrate-from-csv-files-to-tidb.md b/migrate-from-csv-files-to-tidb.md index 38b8afa009f05..5a1f01526f83e 100644 --- a/migrate-from-csv-files-to-tidb.md +++ b/migrate-from-csv-files-to-tidb.md @@ -1,7 +1,6 @@ --- title: Migrate Data from CSV Files to TiDB summary: Learn how to migrate data from CSV files to TiDB. -aliases: ['/docs/dev/tidb-lightning/migrate-from-csv-using-tidb-lightning/','/docs/dev/reference/tools/tidb-lightning/csv/','/tidb/dev/migrate-from-csv-using-tidb-lightning'] --- # Migrate Data from CSV Files to TiDB @@ -22,6 +21,8 @@ Put all the CSV files in the same directory. If you need TiDB Lightning to recog - If a CSV file contains the data for an entire table, name the file `${db_name}.${table_name}.csv`. - If the data of one table is separated into multiple CSV files, append a numeric suffix to these CSV files. For example, `${db_name}.${table_name}.003.csv`. The numeric suffixes can be inconsecutive but must be in ascending order. You also need to add extra zeros before the number to ensure all the suffixes are in the same length. +TiDB Lightning recursively searches for all `.csv` files in this directory and its subdirectories. + ## Step 2. Create the target table schema Because CSV files do not contain schema information, before importing data from CSV files into TiDB, you need to create the target table schema. You can create the target table schema by either of the following two methods: @@ -63,6 +64,8 @@ data-source-dir = "${data-path}" # A local path or S3 path. For example, 's3://m separator = ',' # Delimiter. Can be zero or multiple characters. delimiter = '"' +# Line terminator. By default, \r, \n, and \r\n are all treated as line terminators. +# terminator = "\r\n" # Configures whether the CSV file has a table header. # If this item is set to true, TiDB Lightning uses the first line of the CSV file to parse the corresponding relationship of fields. header = true @@ -104,6 +107,8 @@ In a strict-format CSV file, each field only takes up one line. It must meet the - The delimiter is empty. - Each field does not contain CR (`\r`) or LF (`\n`). +You need to explicitly specify the line terminator `terminator` for a `strict-format` CSV file. + If your CSV file meets the above requirements, you can speed up the import by enabling the `strict-format` mode as follows: ```toml diff --git a/migrate-from-mariadb.md b/migrate-from-mariadb.md new file mode 100644 index 0000000000000..22e56f61958f5 --- /dev/null +++ b/migrate-from-mariadb.md @@ -0,0 +1,345 @@ +--- +title: Migrate Data from MariaDB to TiDB +summary: Learn how to migrate data from MariaDB to TiDB. +--- + +# Migrate Data from MariaDB to TiDB + +This document describes how to migrate data from a MariaDB server installation to a TiDB cluster. + +## Prerequisites + +Choose the right migration strategy: + +- The first strategy is to [dump data with Dumpling and restore data with TiDB Lightning](#dump-data-with-dumpling-and-restore-data-with-tidb-lightning). This works for all versions of MariaDB. The drawback of this strategy is that it needs more downtime. +- The second strategy is to [Replicate data with DM](#replicate-data-with-dm) from MariaDB to TiDB with DM. DM does not support all versions of MariaDB. Supported versions are listed on the [DM Compatibility Catalog](/dm/dm-compatibility-catalog.md#compatibility-catalog-of-tidb-data-migration). + +Besides these two strategies, there might be other strategies available specifically to your situation. For example: + +- Use the functionality of your Object Relational Mapping (ORM) to re-deploy and migrate your data. +- Modify your application to write from both MariaDB and TiDB while the migration is ongoing. + +This document only covers the first two strategies. + +Prepare the following based on the strategy you choose: + +- For the **dump and restore** strategy: + - Install [Dumpling](/dumpling-overview.md) and [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md). + - Make sure you have the [required privileges](/dumpling-overview.md#required-privileges) on the MariaDB server for Dumpling to export data. +- For the **data replication** strategy, set up [Data Migration (DM)](/dm/dm-overview.md). + +## Check compatibility + +TiDB is [compatible with MySQL](/mysql-compatibility.md), and MySQL and MariaDB have a lot of functionality in common. However, there might be MariaDB-specific features that might not be compatible with TiDB that you should be aware of before migrating. + +Besides checking the items in this section, it is recommended that you also check the [Compatibility and Differences](https://mariadb.com/docs/release-notes/community-server/about/compatibility-and-differences) in the MariaDB documentation. + +### Authentication + +The [Security Compatibility with MySQL](/security-compatibility-with-mysql.md) document lists authentication methods that TiDB supports. TiDB does not support a few authentication methods in MariaDB. This means that you might have to create a new password hash for the account or take other specific measures. + +To check what authentication methods are used, you can run the following statement: + +```sql +SELECT + plugin, + COUNT(*) +FROM + mysql.user +GROUP BY + plugin; +``` + +```sql ++-----------------------+----------+ +| plugin | COUNT(*) | ++-----------------------+----------+ +| mysql_native_password | 11 | ++-----------------------+----------+ +1 row in set (0.002 sec) +``` + +### System-versioned tables + +TiDB does not support [system-versioned tables](https://mariadb.com/docs/server/reference/sql-structure/temporal-tables/system-versioned-tables). However, TiDB does support [`AS OF TIMESTAMP`](/as-of-timestamp.md) which might replace some of the use cases of system-versioned tables. + +You can check for affected tables with the following statement: + +```sql +SELECT + TABLE_SCHEMA, + TABLE_NAME +FROM + information_schema.tables +WHERE + TABLE_TYPE='SYSTEM VERSIONED'; +``` + +```sql ++--------------+------------+ +| TABLE_SCHEMA | TABLE_NAME | ++--------------+------------+ +| test | t | ++--------------+------------+ +1 row in set (0.005 sec) +``` + +To remove system versioning, execute the `ALTER TABLE` statement: + +```sql +MariaDB [test]> ALTER TABLE t DROP SYSTEM VERSIONING; +Query OK, 0 rows affected (0.071 sec) +Records: 0 Duplicates: 0 Warnings: 0 +``` + +### Sequences + +Both MariaDB and TiDB support [`CREATE SEQUENCE`](/sql-statements/sql-statement-create-sequence.md). However, it is currently not supported by DM. It is recommended that you do not create, modify, or remove sequences during the migration and test this specifically after migration. + +To check if you are using sequences, execute the following statement: + +```sql +SELECT + TABLE_SCHEMA, + TABLE_NAME +FROM + information_schema.tables +WHERE + TABLE_TYPE='SEQUENCE'; +``` + +```sql ++--------------+------------+ +| TABLE_SCHEMA | TABLE_NAME | ++--------------+------------+ +| test | s1 | ++--------------+------------+ +1 row in set (0.016 sec) +``` + +### Storage engines + +MariaDB offers storage engines for local data such as `InnoDB`, `MyISAM` and `Aria`. While the data format is not directly supported by TiDB, migrating these works fine. However, some engines place data outside of the server, such as the `CONNECT` storage engine and `Spider`. While you can migrate such tables to TiDB, TiDB does not provide the functionality to store data outside of the TiDB cluster. + +To check what storage engines you are using, execute the following statement: + +```sql +SELECT + ENGINE, + COUNT(*) +FROM + information_schema.tables +GROUP BY + ENGINE; +``` + +```sql ++--------------------+----------+ +| ENGINE | COUNT(*) | ++--------------------+----------+ +| NULL | 101 | +| Aria | 38 | +| CSV | 2 | +| InnoDB | 6 | +| MEMORY | 67 | +| MyISAM | 1 | +| PERFORMANCE_SCHEMA | 81 | ++--------------------+----------+ +7 rows in set (0.009 sec) +``` + +### Syntax + +MariaDB supports the `RETURNING` keyword for `DELETE`, `INSERT`, and `REPLACE` statements. TiDB does not support them. You might want to look into your application and query logging to see if it affects your migration. + +### Data types + +MariaDB supports some data types that TiDB does not support, such as `UUID`, `INET4`, and `INET6`. + +To check for these datatypes, execute the following statement: + +```sql +SELECT + TABLE_SCHEMA, + TABLE_NAME, + COLUMN_NAME, + DATA_TYPE +FROM + information_schema.columns +WHERE + DATA_TYPE IN('INET4','INET6','UUID'); +``` + +```sql ++--------------+------------+-------------+-----------+ +| TABLE_SCHEMA | TABLE_NAME | COLUMN_NAME | DATA_TYPE | ++--------------+------------+-------------+-----------+ +| test | u1 | u | uuid | +| test | u1 | i4 | inet4 | +| test | u1 | i6 | inet6 | ++--------------+------------+-------------+-----------+ +3 rows in set (0.026 sec) + +``` + +### Character set and collation + +TiDB does not support the `latin1_swedish_ci` collation that is often used in MariaDB. + +To see what collations TiDB supports, execute this statement on TiDB: + +```sql +SHOW COLLATION; +``` + +```sql ++--------------------+---------+-----+---------+----------+---------+ +| Collation | Charset | Id | Default | Compiled | Sortlen | ++--------------------+---------+-----+---------+----------+---------+ +| ascii_bin | ascii | 65 | Yes | Yes | 1 | +| binary | binary | 63 | Yes | Yes | 1 | +| gbk_bin | gbk | 87 | | Yes | 1 | +| gbk_chinese_ci | gbk | 28 | Yes | Yes | 1 | +| latin1_bin | latin1 | 47 | Yes | Yes | 1 | +| utf8_bin | utf8 | 83 | Yes | Yes | 1 | +| utf8_general_ci | utf8 | 33 | | Yes | 1 | +| utf8_unicode_ci | utf8 | 192 | | Yes | 1 | +| utf8mb4_0900_ai_ci | utf8mb4 | 255 | | Yes | 1 | +| utf8mb4_0900_bin | utf8mb4 | 309 | | Yes | 1 | +| utf8mb4_bin | utf8mb4 | 46 | Yes | Yes | 1 | +| utf8mb4_general_ci | utf8mb4 | 45 | | Yes | 1 | +| utf8mb4_unicode_ci | utf8mb4 | 224 | | Yes | 1 | ++--------------------+---------+-----+---------+----------+---------+ +13 rows in set (0.0012 sec) +``` + +To check what collations the columns of your current tables are using, you can use this statement: + +```sql +SELECT + TABLE_SCHEMA, + COLLATION_NAME, + COUNT(*) +FROM + information_schema.columns +GROUP BY + TABLE_SCHEMA, COLLATION_NAME +ORDER BY + COLLATION_NAME; +``` + +```sql ++--------------------+--------------------+----------+ +| TABLE_SCHEMA | COLLATION_NAME | COUNT(*) | ++--------------------+--------------------+----------+ +| sys | NULL | 562 | +| test | NULL | 14 | +| mysql | NULL | 84 | +| performance_schema | NULL | 892 | +| information_schema | NULL | 421 | +| mysql | latin1_swedish_ci | 34 | +| performance_schema | utf8mb3_bin | 38 | +| mysql | utf8mb3_bin | 61 | +| sys | utf8mb3_bin | 40 | +| information_schema | utf8mb3_general_ci | 375 | +| performance_schema | utf8mb3_general_ci | 244 | +| sys | utf8mb3_general_ci | 386 | +| mysql | utf8mb3_general_ci | 67 | +| mysql | utf8mb4_bin | 8 | ++--------------------+--------------------+----------+ +14 rows in set (0.045 sec) +``` + +See also [Character Set and Collation](/character-set-and-collation.md). + +## Dump data with Dumpling and restore data with TiDB Lightning + +This method assumes that you take your application offline, migrate the data, and then re-configure your application to use the migrated data. + +> **Note:** +> +> It is strongly recommended to first do this on a test or development instance of your application before doing it in production. This is both to check for possible compatibility issues as to get insight into how much time the migration will take. + +Perform the following steps to migrate data from MariaDB to TiDB: + +1. Stop your application. Take your application offline. This ensures there are no modifications made to the data in MariaDB during or after the migration. + +2. Dump data in MariaDB with the [`tiup dumpling`](/dumpling-overview.md#use-dumpling-to-export-data) command. + + ```shell + tiup dumpling --port 3306 --host 127.0.0.1 --user root --password secret -F 256MB -o /data/backup + ``` + +3. Restore the data by using the `tiup tidb-lightning` command. For more information about how to configure TiDB Lightning and how to run it, see [Get Started with TiDB Lightning](/get-started-with-tidb-lightning.md). + +4. Migrate user accounts and permissions. For more information about how to migrate your users and permissions, see [Export users and grants](#export-users-and-grants). + +5. Reconfigure your application. You need to change the application configuration so that it can connect to the TiDB server. + +6. Clean up. Once you have verified that the migration is successful you can make a final backup of the data in MariaDB and stop the server. This also means you can remove tools such as TiUP, Dumpling, and TiDB Lightning. + +## Replicate data with DM + +This method assumes you would set up replication, stop your application and wait for the replication to catch up, and then re-configure your application to use TiDB. + +To use DM, you need to deploy a set of DM services either with a [TiUP cluster](/dm/deploy-a-dm-cluster-using-tiup.md) or with [TiDB Operator](/tidb-operator-overview.md). After that, use `dmctl` to configure the DM services. + +> **Note:** +> +> It is strongly recommended to first do this on a test or development instance of your application before doing it in production. This is both to check for possible compatibility issues as to get insight into how much time the migration will take. + +### Step 1. Prepare + +Make sure that binlogs are enabled on MariaDB and that the `binlog_format` is set to `ROW`. It is also recommended to set `binlog_annotate_row_events=OFF` and `log_bin_compress=OFF`. + +You also need an account with the `SUPER` permission or with the `BINLOG MONITOR` and `REPLICATION MASTER ADMIN` permissions. This account also needs read permission for the schemas you are going to migrate. + +If you are not using an account with the `SUPER` permission, then you might have to add the following to the DM configuration, because TiDB does not yet know how to check for MariaDB specific permissions. + +```yaml +ignore-checking-items: ["replication_privilege"] +``` + +Before you use DM to migrate data from upstream to downstream, a precheck helps detect errors in the upstream database configurations and ensures that the migration goes smoothly. For more information, see [Migration Task Precheck](/dm/dm-precheck.md) + +### Step 2. Replicate data + +Follow the [Quick Start Guide for TiDB Data Migration](/dm/quick-start-with-dm.md) to replicate your data from MariaDB to TiDB. + +Note that it is not required to first copy the initial data as you would do with MariaDB to MariaDB replication, DM will do this for you. + +### Step 3. Migrate user accounts and permissions + +See [Export users and grants](#export-users-and-grants) for how to migrate your users and permissions. + +### Step 4. Test your data + +Once your data is replicated, you can run read-only queries on it to validate it. For more information, see [Test your application](#test-your-application). + +### Step 5. Switch over + +To switch over to TiDB, you need to do the following: + +1. Stop your application. +2. Monitor the replication delay, which should go to 0 seconds. +3. Change the configuration of your application so that it connects to TiDB and start it again. + +To check for replication delay, run [`query-status `](/dm/dm-query-status.md#detailed-query-result) via `dmctl` and check for `"synced: true"` in the `subTaskStatus`. + +### Step 6. Clean up + +Once you have verified that the migration is successful, you can make a final backup of the data in MariaDB and stop the server. It also means you can stop and remove the DM cluster. + +## Export users and grants + +You can use [`pt-show-grants`](https://docs.percona.com/percona-toolkit/pt-show-grants.html). It is part of the Percona Toolkit to export users and grants from MariaDB and load these into TiDB. + +## Test your application + +While it is possible to use generic tools such as `sysbench` for testing, it is highly recommended to test some specific features of your application. For example, run a copy of your application against a TiDB cluster with a temporary copy of your data. + +Such a test makes sure your application compatibility and performance with TiDB is verified. You need to monitor the log files of your application and TiDB to see if there are any warnings that might need to be addressed. Make sure that the database driver that your application is using (for example MySQL Connector/J for Java based applications) is tested. You might want to use an application such as JMeter to put some load on your application if needed. + +## Validate data + +You can use [sync-diff-inspector](/sync-diff-inspector/sync-diff-inspector-overview.md) to validate if the data in MariaDB and TiDB are identical. diff --git a/migrate-from-parquet-files-to-tidb.md b/migrate-from-parquet-files-to-tidb.md index 0ca90cb019469..ef37bb9d10fd1 100644 --- a/migrate-from-parquet-files-to-tidb.md +++ b/migrate-from-parquet-files-to-tidb.md @@ -41,7 +41,7 @@ Each table in Hive can be exported to parquet files by annotating `STORED AS PAR DROP TABLE temp; ``` -3. The parquet files exported from Hive might not have the `.parquet` suffix and cannot be correctly identified by TiDB Lightning. Therefore, before importing the files, you need to rename the exported files and add the `.parquet` suffix. +3. The parquet files exported from Hive might not have the `.parquet` suffix and cannot be correctly identified by TiDB Lightning. Therefore, before importing the files, you need to rename the exported files and add the `.parquet` suffix to change the full filename to a format that TiDB Lightning recognizes, for example, `${db_name}. ${table_name}.parquet`. For more information about file types and patterns, see [TiDB Lightning Data Sources](/tidb-lightning/tidb-lightning-data-source.md). You can also match data files by setting correct [customized expressions](/tidb-lightning/tidb-lightning-data-source.md#match-customized-files). 4. Put all the parquet files in a unified directory, for example, `/data/my_datasource/` or `s3://my-bucket/sql-backup`. TiDB Lightning will recursively search for all `.parquet` files in this directory and its subdirectories. diff --git a/migrate-from-sql-files-to-tidb.md b/migrate-from-sql-files-to-tidb.md index 877656adbeb24..1f43d9331d68d 100644 --- a/migrate-from-sql-files-to-tidb.md +++ b/migrate-from-sql-files-to-tidb.md @@ -1,7 +1,6 @@ --- title: Migrate Data from SQL Files to TiDB summary: Learn how to migrate data from SQL files to TiDB. -aliases: ['/docs/dev/migrate-from-mysql-mydumper-files/','/tidb/dev/migrate-from-mysql-mydumper-files/','/tidb/dev/migrate-from-mysql-dumpling-files'] --- # Migrate Data from SQL Files to TiDB diff --git a/migrate-from-tidb-to-mysql.md b/migrate-from-tidb-to-mysql.md index b676bbee47888..85b4220e0385a 100644 --- a/migrate-from-tidb-to-mysql.md +++ b/migrate-from-tidb-to-mysql.md @@ -56,7 +56,7 @@ After setting up the environment, you can use [Dumpling](/dumpling-overview.md) 1. Disable Garbage Collection (GC). - To ensure that newly written data is not deleted during incremental migration, you should disable GC for the upstream cluster before exporting full data. In this way, history data is not deleted. + To ensure that newly written data is not deleted during incremental migration, you should disable GC for the upstream cluster before exporting full data. In this way, history data is not deleted. For TiDB v4.0.0 and later versions, Dumpling might [automatically adjust the GC safe point to block GC](/dumpling-overview.md#manually-set-the-tidb-gc-time). Nevertheless, manually disabling GC is still necessary because the GC process might begin after Dumpling exits, leading to the failure of incremental changes migration. Run the following command to disable GC: @@ -160,7 +160,7 @@ After setting up the environment, you can use [Dumpling](/dumpling-overview.md) In the upstream cluster, run the following command to create a changefeed from the upstream to the downstream clusters: ```shell - tiup ctl:v cdc changefeed create --server=http://127.0.0.1:8300 --sink-uri="mysql://root:@127.0.0.1:3306" --changefeed-id="upstream-to-downstream" --start-ts="434217889191428107" + tiup cdc:v cli changefeed create --server=http://127.0.0.1:8300 --sink-uri="mysql://root:@127.0.0.1:3306" --changefeed-id="upstream-to-downstream" --start-ts="434217889191428107" ``` In this command, the parameters are as follows: diff --git a/migrate-from-tidb-to-tidb.md b/migrate-from-tidb-to-tidb.md index 18b9f7bad2bfe..e0d050feed3b2 100644 --- a/migrate-from-tidb-to-tidb.md +++ b/migrate-from-tidb-to-tidb.md @@ -99,13 +99,13 @@ This document exemplifies the whole migration process and contains the following ## Step 2. Migrate full data -After setting up the environment, you can use the backup and restore functions of [BR](https://github.com/pingcap/tidb/tree/master/br) to migrate full data. BR can be started in [three ways](/br/br-use-overview.md#deploy-and-use-br). In this document, we use the SQL statements, `BACKUP` and `RESTORE`. +After setting up the environment, you can use the backup and restore functions of [BR](https://github.com/pingcap/tidb/tree/release-7.5/br) to migrate full data. BR can be started in [three ways](/br/br-use-overview.md#deploy-and-use-br). In this document, we use the SQL statements, `BACKUP` and `RESTORE`. > **Note:** > -> In production clusters, performing a backup with GC disabled might affect cluster performance. It is recommended that you back up data in off-peak hours, and set `RATE_LIMIT` to a proper value to avoid performance degradation. -> -> If the versions of the upstream and downstream clusters are different, you should check [BR compatibility](/br/backup-and-restore-overview.md#before-you-use). In this document, we assume that the upstream and downstream clusters are the same version. +> - `BACKUP` and `RESTORE` SQL statements are experimental. It is not recommended that you use them in the production environment. They might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. +> - In production clusters, performing a backup with GC disabled might affect cluster performance. It is recommended that you back up data in off-peak hours, and set `RATE_LIMIT` to a proper value to avoid performance degradation. +> - If the versions of the upstream and downstream clusters are different, you should check [BR compatibility](/br/backup-and-restore-overview.md#before-you-use). In this document, we assume that the upstream and downstream clusters are the same version. 1. Disable GC. diff --git a/migrate-large-mysql-shards-to-tidb.md b/migrate-large-mysql-shards-to-tidb.md index b5f3513e322f6..887c06776f550 100644 --- a/migrate-large-mysql-shards-to-tidb.md +++ b/migrate-large-mysql-shards-to-tidb.md @@ -281,7 +281,7 @@ name: task-test # The name of the task. Should be globally unique. task-mode: incremental # The mode of the task. "incremental" means full data migration is skipped and only incremental replication is performed. # Required for incremental replication from sharded tables. By default, the "pessimistic" mode is used. # If you have a deep understanding of the principles and usage limitations of the optimistic mode, you can also use the "optimistic" mode. -# For more information, see [Merge and Migrate Data from Sharded Tables](https://docs.pingcap.com/zh/tidb/dev/feature-shard-merge/). +# For more information, see [Merge and Migrate Data from Sharded Tables](https://docs.pingcap.com/tidb/dev/feature-shard-merge/). shard-mode: "pessimistic" diff --git a/migrate-large-mysql-to-tidb.md b/migrate-large-mysql-to-tidb.md index c7d86294e7723..60cb8bf52021b 100644 --- a/migrate-large-mysql-to-tidb.md +++ b/migrate-large-mysql-to-tidb.md @@ -1,9 +1,9 @@ --- -title: Migrate MySQL of Large Datasets to TiDB -summary: Learn how to migrate MySQL of large datasets to TiDB. +title: Migrate Large Datasets from MySQL to TiDB +summary: Learn how to migrate large datasets from MySQL to TiDB. --- -# Migrate MySQL of Large Datasets to TiDB +# Migrate Large Datasets from MySQL to TiDB When the data volume to be migrated is small, you can easily [use DM to migrate data](/migrate-small-mysql-to-tidb.md), both for full migration and incremental replication. However, because DM imports data at a slow speed (30~50 GiB/h), when the data volume is large, the migration might take a long time. "Large datasets" in this document usually mean data around one TiB or more. @@ -29,16 +29,35 @@ This document describes how to perform the full migration using Dumpling and TiD - During the import, TiDB Lightning needs temporary space to store the sorted key-value pairs. The disk space should be enough to hold the largest single table from the data source. - If the full data volume is large, you can increase the binlog storage time in the upstream. This is to ensure that the binlogs are not lost during the incremental replication. -**Note**: It is difficult to calculate the exact data volume exported by Dumpling from MySQL, but you can estimate the data volume by using the following SQL statement to summarize the `data-length` field in the `information_schema.tables` table: - -{{< copyable "" >}} +**Note**: It is difficult to calculate the exact data volume exported by Dumpling from MySQL, but you can estimate the data volume by using the following SQL statement to summarize the `DATA_LENGTH` field in the `information_schema.tables` table: ```sql -/* Calculate the size of all schemas, in MiB. Replace ${schema_name} with your schema name. */ -SELECT table_schema,SUM(data_length)/1024/1024 AS data_length,SUM(index_length)/1024/1024 AS index_length,SUM(data_length+index_length)/1024/1024 AS SUM FROM information_schema.tables WHERE table_schema = "${schema_name}" GROUP BY table_schema; - -/* Calculate the size of the largest table, in MiB. Replace ${schema_name} with your schema name. */ -SELECT table_name,table_schema,SUM(data_length)/1024/1024 AS data_length,SUM(index_length)/1024/1024 AS index_length,SUM(data_length+index_length)/1024/1024 AS SUM from information_schema.tables WHERE table_schema = "${schema_name}" GROUP BY table_name,table_schema ORDER BY SUM DESC LIMIT 5; +-- Calculate the size of all schemas +SELECT + TABLE_SCHEMA, + FORMAT_BYTES(SUM(DATA_LENGTH)) AS 'Data Size', + FORMAT_BYTES(SUM(INDEX_LENGTH)) 'Index Size' +FROM + information_schema.tables +GROUP BY + TABLE_SCHEMA; + +-- Calculate the 5 largest tables +SELECT + TABLE_NAME, + TABLE_SCHEMA, + FORMAT_BYTES(SUM(data_length)) AS 'Data Size', + FORMAT_BYTES(SUM(index_length)) AS 'Index Size', + FORMAT_BYTES(SUM(data_length+index_length)) AS 'Total Size' +FROM + information_schema.tables +GROUP BY + TABLE_NAME, + TABLE_SCHEMA +ORDER BY + SUM(DATA_LENGTH+INDEX_LENGTH) DESC +LIMIT + 5; ``` ### Disk space for the target TiKV cluster @@ -121,18 +140,19 @@ The target TiKV cluster must have enough disk space to store the imported data. For more information on TiDB Lightning configuration, refer to [TiDB Lightning Configuration](/tidb-lightning/tidb-lightning-configuration.md). -2. Start the import by running `tidb-lightning`. If you launch the program directly in the command line, the process might exit unexpectedly after receiving a SIGHUP signal. In this case, it is recommended to run the program using a `nohup` or `screen` tool. For example: +2. Start the import by running `tidb-lightning`. If you launch the program directly in the command line, the process might exit unexpectedly after receiving a SIGHUP signal. It is not recommended to directly use `nohup` to start a process from the command line. Instead, edit the following script content, for example: If you import data from S3, pass the SecretKey and AccessKey that have access to the S3 storage path as environment variables to the TiDB Lightning node. You can also read the credentials from `~/.aws/credentials`. - {{< copyable "shell-regular" >}} - ```shell + #!/bin/bash export AWS_ACCESS_KEY_ID=${access_key} export AWS_SECRET_ACCESS_KEY=${secret_key} nohup tiup tidb-lightning -config tidb-lightning.toml > nohup.out 2>&1 & ``` + Then use the script to start TiDB Lightning. + 3. After the import starts, you can check the progress of the import by one of the following methods: - `grep` the keyword `progress` in the log. The progress is updated every 5 minutes by default. diff --git a/migrate-small-mysql-shards-to-tidb.md b/migrate-small-mysql-shards-to-tidb.md index 36a08188b0057..1dada69d32c66 100644 --- a/migrate-small-mysql-shards-to-tidb.md +++ b/migrate-small-mysql-shards-to-tidb.md @@ -1,7 +1,6 @@ --- title: Migrate and Merge MySQL Shards of Small Datasets to TiDB summary: Learn how to migrate and merge small datasets of shards from MySQL to TiDB. -aliases: ['/tidb/dev/usage-scenario-shard-merge/', '/tidb/dev/usage-scenario-simple-migration/'] --- # Migrate and Merge MySQL Shards of Small Datasets to TiDB @@ -118,7 +117,7 @@ name: "shard_merge" # The name of the task. Should be globally uni task-mode: all # Required for the MySQL shards. By default, the "pessimistic" mode is used. # If you have a deep understanding of the principles and usage limitations of the optimistic mode, you can also use the "optimistic" mode. -# For more information, see [Merge and Migrate Data from Sharded Tables](https://docs.pingcap.com/tidb/dev/feature-shard-merge/) +# For more information, see [Merge and Migrate Data from Sharded Tables](https://docs.pingcap.com/tidb/v7.5/feature-shard-merge/) shard-mode: "pessimistic" meta-schema: "dm_meta" # A schema will be created in the downstream database to store the metadata ignore-checking-items: ["auto_increment_ID"] # In this example, there are auto-incremental primary keys upstream, so you do not need to check this item. diff --git a/migrate-small-mysql-to-tidb.md b/migrate-small-mysql-to-tidb.md index 62cd52ad12a22..23078f98c4245 100644 --- a/migrate-small-mysql-to-tidb.md +++ b/migrate-small-mysql-to-tidb.md @@ -1,12 +1,11 @@ --- -title: Migrate MySQL of Small Datasets to TiDB -summary: Learn how to migrate MySQL of small datasets to TiDB. -aliases: ['/tidb/dev/usage-scenario-incremental-migration/'] +title: Migrate Small Datasets from MySQL to TiDB +summary: Learn how to migrate small datasets from MySQL to TiDB. --- -# Migrate MySQL of Small Datasets to TiDB +# Migrate Small Datasets from MySQL to TiDB -This document describes how to use TiDB Data Migration (DM) to migrate MySQL of small datasets to TiDB in the full migration mode and incremental replication mode. "Small datasets" in this document mean data size less than 1 TiB. +This document describes how to use TiDB Data Migration (DM) to migrate small datasets from MySQL to TiDB in the full migration mode and incremental replication mode. "Small datasets" in this document mean data size less than 1 TiB. The migration speed varies from 30 GB/h to 50 GB/h, depending on multiple factors such as the number of indexes in the table schema, hardware, and network environment. @@ -137,8 +136,8 @@ To view the historical status of the migration task and other internal metrics, If you have deployed Prometheus, Alertmanager, and Grafana when deploying DM using TiUP, you can access Grafana using the IP address and port specified during the deployment. You can then select the DM dashboard to view DM-related monitoring metrics. -- The log directory of DM-master: specified by the DM-master process parameter `--log-file`. If you have deployd DM using TiUP, the log directory is `/dm-deploy/dm-master-8261/log/` by default. -- The log directory of DM-worker: specified by the DM-worker process parameter `--log-file`. If you have deployd DM using TiUP, the log directory is `/dm-deploy/dm-worker-8262/log/` by default. +- The log directory of DM-master: specified by the DM-master process parameter `--log-file`. If you have deployed DM using TiUP, the log directory is `/dm-deploy/dm-master-8261/log/` by default. +- The log directory of DM-worker: specified by the DM-worker process parameter `--log-file`. If you have deployed DM using TiUP, the log directory is `/dm-deploy/dm-worker-8262/log/` by default. ## What's next diff --git a/migrate-with-more-columns-downstream.md b/migrate-with-more-columns-downstream.md index e6fac2dafc5fb..10fd50c2f84de 100644 --- a/migrate-with-more-columns-downstream.md +++ b/migrate-with-more-columns-downstream.md @@ -1,15 +1,14 @@ --- title: Migrate Data to a Downstream TiDB Table with More Columns summary: Learn how to migrate data to a downstream TiDB table with more columns than the corresponding upstream table. -aliases: ['/tidb/dev/usage-scenario-downstream-more-columns/'] --- # Migrate Data to a Downstream TiDB Table with More Columns This document provides the additional steps to be taken when you migrate data to a downstream TiDB table with more columns than the corresponding upstream table. For regular migration steps, see the following migration scenarios: -- [Migrate MySQL of Small Datasets to TiDB](/migrate-small-mysql-to-tidb.md) -- [Migrate MySQL of Large Datasets to TiDB](/migrate-large-mysql-to-tidb.md) +- [Migrate Small Datasets from MySQL to TiDB](/migrate-small-mysql-to-tidb.md) +- [Migrate Large Datasets from MySQL to TiDB](/migrate-large-mysql-to-tidb.md) - [Migrate and Merge MySQL Shards of Small Datasets to TiDB](/migrate-small-mysql-shards-to-tidb.md) - [Migrate and Merge MySQL Shards of Large Datasets to TiDB](/migrate-large-mysql-shards-to-tidb.md) @@ -107,5 +106,5 @@ In such cases, you can use the `binlog-schema` command to set a table schema for {{< copyable "shell-regular" >}} ``` - tiup dmctl --master-addr ${advertise-addr} query-status resume-task ${task-name} + tiup dmctl --master-addr ${advertise-addr} query-status ${task-name} ``` diff --git a/migrate-with-pt-ghost.md b/migrate-with-pt-ghost.md index 90d63a7b281d2..4505e7a38c3f6 100644 --- a/migrate-with-pt-ghost.md +++ b/migrate-with-pt-ghost.md @@ -7,12 +7,12 @@ summary: Learn how to use DM to replicate incremental data from databases that u In production scenarios, table locking during DDL execution can block the reads from or writes to the database to a certain extent. Therefore, online DDL tools are often used to execute DDLs to minimize the impact on reads and writes. Common DDL tools are [gh-ost](https://github.com/github/gh-ost) and [pt-osc](https://www.percona.com/doc/percona-toolkit/3.0/pt-online-schema-change.html). -When using DM to migrate data from MySQL to TiDB, you can enbale `online-ddl` to allow collaboration of DM and gh-ost or pt-osc. +When using DM to migrate data from MySQL to TiDB, you can enable `online-ddl` to allow collaboration of DM and gh-ost or pt-osc. For the detailed replication instructions, refer to the following documents by scenarios: -- [Migrate MySQL of Small Datasets to TiDB](/migrate-small-mysql-to-tidb.md) -- [Migrate MySQL of Large Datasets to TiDB](/migrate-large-mysql-to-tidb.md) +- [Migrate Small Datasets from MySQL to TiDB](/migrate-small-mysql-to-tidb.md) +- [Migrate Large Datasets from MySQL to TiDB](/migrate-large-mysql-to-tidb.md) - [Migrate and Merge MySQL Shards of Small Datasets to TiDB](/migrate-small-mysql-shards-to-tidb.md) - [Migrate and Merge MySQL Shards of Large Datasets to TiDB](/migrate-large-mysql-shards-to-tidb.md) diff --git a/migration-overview.md b/migration-overview.md index 6cf6e549ce19f..326eced1fe5c1 100644 --- a/migration-overview.md +++ b/migration-overview.md @@ -30,13 +30,9 @@ When you migrate data from Aurora to a TiDB cluster deployed on AWS, your data m ## Migrate data from MySQL to TiDB -If cloud storage (S3) service is not used, the network connectivity is good, and the network latency is low, you can use the following method to migrate data from MySQL to TiDB. +If cloud storage (S3) service is not used, the network connectivity is good, and the network latency is low, you can follow instructions in [Migrate Small Datasets from MySQL to TiDB](/migrate-small-mysql-to-tidb.md) to migrate data from MySQL to TiDB. -- [Migrate MySQL of Small Datasets to TiDB](/migrate-small-mysql-to-tidb.md) - -If you have a high demand on migration speed, or if the data size is large (for example, larger than 1 TiB), and you do not allow other applications to write to TiDB during the migration period, you can use TiDB Lightning to quickly import data. Then, you can use DM to replicate incremental data (binlog) based on your application needs. - -- [Migrate MySQL of Large Datasets to TiDB](/migrate-large-mysql-to-tidb.md) +If you have a high demand on migration speed, or if the data size is large (for example, larger than 1 TiB), and you do not allow other applications to write to TiDB during the migration period, you can use TiDB Lightning to quickly import data. Then, you can use DM to replicate incremental data (binlog) based on your application needs. See [Migrate Large Datasets from MySQL to TiDB](/migrate-large-mysql-to-tidb.md). ## Migrate and merge MySQL shards into TiDB diff --git a/migration-tools.md b/migration-tools.md index 39fca56e6c02e..104346177e4e8 100644 --- a/migration-tools.md +++ b/migration-tools.md @@ -73,7 +73,7 @@ This document introduces the user scenarios, supported upstreams and downstreams - Support backing up data to an external storage for disaster recovery - **Limitation**: - When BR restores data to the upstream cluster of TiCDC or Drainer, the restored data cannot be replicated to the downstream by TiCDC or Drainer. - - BR supports operations only between clusters that have the same `new_collations_enabled_on_first_bootstrap` value. + - BR supports operations only between clusters that have the same `new_collation_enabled` value in the `mysql.tidb` table. ## [sync-diff-inspector](/sync-diff-inspector/sync-diff-inspector-overview.md) diff --git a/minimal-deployment-topology.md b/minimal-deployment-topology.md index 986ec1bde8050..8adc020042613 100644 --- a/minimal-deployment-topology.md +++ b/minimal-deployment-topology.md @@ -1,7 +1,6 @@ --- title: Minimal Deployment Topology summary: Learn the minimal deployment topology of TiDB clusters. -aliases: ['/docs/dev/minimal-deployment-topology/'] --- # Minimal Deployment Topology @@ -17,6 +16,10 @@ This document describes the minimal deployment topology of TiDB clusters. | TiKV | 3 | 16 VCore 32 GiB
2 TiB (NVMe SSD) for storage | 10.0.1.7
10.0.1.8
10.0.1.9 | Default port
Global directory configuration | | Monitoring & Grafana | 1 | 4 VCore 8 GiB
500 GiB (SSD) for storage | 10.0.1.10 | Default port
Global directory configuration | +> **Note:** +> +> The IP addresses of the instances are given as examples only. In your actual deployment, replace the IP addresses with your actual IP addresses. + ### Topology templates - [The simple template for the minimal topology](https://github.com/pingcap/docs/blob/master/config-templates/simple-mini.yaml) diff --git a/multi-data-centers-in-one-city-deployment.md b/multi-data-centers-in-one-city-deployment.md index 5f67a00b6040f..456aebb04fbe9 100644 --- a/multi-data-centers-in-one-city-deployment.md +++ b/multi-data-centers-in-one-city-deployment.md @@ -1,7 +1,6 @@ --- title: Multiple Availability Zones in One Region Deployment summary: Learn the deployment solution to multiple availability zones in one region. -aliases: ['/docs/dev/how-to/deploy/geographic-redundancy/overview/','/docs/dev/geo-redundancy-deployment/','/tidb/dev/geo-redundancy-deployment'] --- # Multiple Availability Zones in One Region Deployment diff --git a/mysql-compatibility.md b/mysql-compatibility.md index 0716f33b423a7..804588521b5c9 100644 --- a/mysql-compatibility.md +++ b/mysql-compatibility.md @@ -1,7 +1,6 @@ --- title: MySQL Compatibility summary: Learn about the compatibility of TiDB with MySQL, and the unsupported and different features. -aliases: ['/docs/dev/mysql-compatibility/','/docs/dev/reference/mysql-compatibility/'] --- # MySQL Compatibility @@ -73,6 +72,7 @@ You can try out TiDB features on [TiDB Playground](https://play.tidbcloud.com/?u + Descending Index [#2519](https://github.com/pingcap/tidb/issues/2519) + `SKIP LOCKED` syntax [#18207](https://github.com/pingcap/tidb/issues/18207) + Lateral derived tables [#40328](https://github.com/pingcap/tidb/issues/40328) ++ JOIN ON subquery [#11414](https://github.com/pingcap/tidb/issues/11414) ## Differences from MySQL @@ -138,13 +138,13 @@ As shown, because of the shared allocator, the `id` increments by 2 each time. T -TiDB utilizes a combination of [Prometheus and Grafana](/tidb-monitoring-api.md) for storing and querying performance monitoring metrics. In TiDB, performance schema tables do not return any results. +TiDB utilizes a combination of [Prometheus and Grafana](/tidb-monitoring-api.md) for storing and querying performance monitoring metrics. In TiDB, most [performance schema tables](/performance-schema/performance-schema.md) do not return any results. -To check performance metrics in TiDB Cloud, you can either check the cluster overview page on the TiDB Cloud console or use [third-party monitoring integrations](/tidb-cloud/third-party-monitoring-integrations.md). Performance schema tables return empty results in TiDB. +To check performance metrics in TiDB Cloud, you can either check the cluster overview page in the TiDB Cloud console or use [third-party monitoring integrations](/tidb-cloud/third-party-monitoring-integrations.md). Most [performance schema tables](/performance-schema/performance-schema.md) return empty results in TiDB. @@ -277,6 +277,10 @@ The following column types are supported by MySQL but **not** by TiDB: - `SQL_TSI_*` (includes SQL_TSI_MONTH, SQL_TSI_WEEK, SQL_TSI_DAY, SQL_TSI_HOUR, SQL_TSI_MINUTE, and SQL_TSI_SECOND, but excludes SQL_TSI_YEAR) +### Regular expressions + +For information about TiDB regular expression compatibility with MySQL, including `REGEXP_INSTR()`, `REGEXP_LIKE()`, `REGEXP_REPLACE()`, and `REGEXP_SUBSTR()`, see [Regular expression compatibility with MySQL](/functions-and-operators/string-functions.md#regular-expression-compatibility-with-mysql). + ### Incompatibility due to deprecated features TiDB does not implement specific features deprecated in MySQL, including: diff --git a/mysql-schema.md b/mysql-schema.md index 0f55e80c280b7..006e99a5a2999 100644 --- a/mysql-schema.md +++ b/mysql-schema.md @@ -1,7 +1,6 @@ --- title: mysql Schema summary: Learn about the TiDB system tables. -aliases: ['/docs/dev/system-tables/system-table-overview/','/docs/dev/reference/system-databases/mysql/','/tidb/dev/system-table-overview/'] --- # `mysql` Schema @@ -22,6 +21,15 @@ These system tables contain grant information about user accounts and their priv - `global_priv`: the authentication information based on certificates - `role_edges`: the relationship between roles +## Cluster status system tables + +* The `tidb` table contains some global information about TiDB: + + * `bootstrapped`: whether the TiDB cluster has been initialized. Note that this value is read-only and cannot be modified. + * `tidb_server_version`: the version information of TiDB when it is initialized. Note that this value is read-only and cannot be modified. + * `system_tz`: the system time zone of TiDB. + * `new_collation_enabled`: whether TiDB has enabled the [new framework for collations](/character-set-and-collation.md#new-framework-for-collations). Note that this value is read-only and cannot be modified. + ## Server-side help system tables Currently, the `help_topic` is NULL. @@ -35,6 +43,9 @@ Currently, the `help_topic` is NULL. - `stats_extended`: extended statistics, such as the order correlation between columns - `stats_feedback`: the query feedback of statistics - `stats_fm_sketch`: the FMSketch distribution of the histogram of the statistics column +- `stats_table_locked`: information about the locked statistics +- `stats_meta_history`: the meta information in the historical statistics +- `stats_history`: the other information in the historical statistics - `analyze_options`: the default `analyze` options for each table - `column_stats_usage`: the usage of column statistics - `schema_index_usage`: the usage of indexes @@ -45,11 +56,16 @@ Currently, the `help_topic` is NULL. - `bind_info`: the binding information of execution plans - `capture_plan_baselines_blacklist`: the blocklist for the automatic binding of the execution plan +## System tables related to PLAN REPLAYER + +- `plan_replayer_status`: the [`PLAN REPLAYER CAPTURE`](https://docs.pingcap.com/tidb/stable/sql-plan-replayer#use-plan-replayer-capture) tasks registered by the user +- `plan_replayer_task`: the results of [`PLAN REPLAYER CAPTURE`](https://docs.pingcap.com/tidb/stable/sql-plan-replayer#use-plan-replayer-capture) tasks + ## GC worker system tables > **Note:** > -> The GC worker system tables are only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> The GC worker system tables are only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). - `gc_delete_range`: the KV range to be deleted - `gc_delete_range_done`: the deleted KV range @@ -60,10 +76,6 @@ Currently, the `help_topic` is NULL. ## TTL related system tables -> **Note:** - -> The TTL related system tables are not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. - * `tidb_ttl_table_status`: the previously executed TTL job and ongoing TTL job for all TTL tables * `tidb_ttl_task`: the current ongoing TTL subtasks * `tidb_ttl_job_history`: the execution history of TTL tasks in the last 90 days @@ -76,8 +88,22 @@ Currently, the `help_topic` is NULL. ## System tables related to metadata locks -* `tidb_mdl_view`:a view of metadata locks. You can use it to view information about the currently blocked DDL statements -* `tidb_mdl_info`:used internally by TiDB to synchronize metadata locks across nodes +* `tidb_mdl_view`: a view of metadata locks. You can use it to view information about the currently blocked DDL statements +* `tidb_mdl_info`: used internally by TiDB to synchronize metadata locks across nodes + +## System tables related to DDL statements + +* `tidb_ddl_history`: the history records of DDL statements +* `tidb_ddl_job`: the metadata of DDL statements that are currently being executed by TiDB +* `tidb_ddl_reorg`: the metadata of physical DDL statements (such as adding indexes) that are currently being executed by TiDB + +## System tables related to TiDB Distributed eXecution Framework (DXF) + +* `dist_framework_meta`: the metadata of the Distributed eXecution Framework (DXF) task scheduler +* `tidb_global_task`: the metadata of the current DXF task +* `tidb_global_task_history`: the metadata of the historical DXF tasks, including both succeeded and failed tasks +* `tidb_background_subtask`: the metadata of the current DXF subtask +* `tidb_background_subtask_history`: the metadata of the historical DXF subtasks ## Miscellaneous system tables @@ -85,15 +111,14 @@ Currently, the `help_topic` is NULL. > **Note:** > -> The `tidb`, `expr_pushdown_blacklist`, `opt_rule_blacklist`, `table_cache_meta`, `tidb_import_jobs`, and `tidb_timers` system tables are only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> The `tidb`, `expr_pushdown_blacklist`, `opt_rule_blacklist`, `table_cache_meta`, `tidb_import_jobs`, and `tidb_timers` system tables are only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). - `GLOBAL_VARIABLES`: global system variable table -- `tidb`: to record the version information when TiDB executes `bootstrap` - `expr_pushdown_blacklist`: the blocklist for expression pushdown - `opt_rule_blacklist`: the blocklist for logical optimization rules -- `table_cache_meta`: the metadata of cached tables - `tidb_import_jobs`: the job information of [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) - `tidb_timers`: the metadata of internal timers +- `advisory_locks`: information related to [Locking functions](/functions-and-operators/locking-functions.md)
diff --git a/non-transactional-dml.md b/non-transactional-dml.md index 61425496216c7..bcace30d98378 100644 --- a/non-transactional-dml.md +++ b/non-transactional-dml.md @@ -289,7 +289,7 @@ The following are hard restrictions on non-transactional DML statements. If thes - The shard column must be indexed. The index can be a single-column index, or the first column of a joint index. - Must be used in the [`autocommit`](/system-variables.md#autocommit) mode. - Cannot be used when batch-dml is enabled. -- Cannot be used when [`tidb_snapshot`](/read-historical-data.md#operation flow) is set. +- Cannot be used when [`tidb_snapshot`](/read-historical-data.md) is set. - Cannot be used with the `prepare` statement. - `ENUM`, `BIT`, `SET`, `JSON` types are not supported as the shard columns. - Not supported for [temporary tables](/temporary-tables.md). @@ -328,6 +328,65 @@ Non-transactional DML statements are not yet a replacement for all batch-dml usa ## Common issues +### Executing a multiple table joins statement results in the `Unknown column xxx in 'where clause'` error + +This error occurs when the `WHERE` clause concatenated in a query involves tables other than the table in which the [shard column](#parameter-description) is defined. For example, in the following SQL statement, the shard column is `t2.id` and it is defined in table `t2`, but the `WHERE` clause involves table `t2` and `t3`. + +```sql +BATCH ON test.t2.id LIMIT 1 +INSERT INTO t +SELECT t2.id, t2.v, t3.id FROM t2, t3 WHERE t2.id = t3.id +``` + +```sql +(1054, "Unknown column 't3.id' in 'where clause'") +``` + +If the error occurs, you can print the query statement for confirmation by using `DRY RUN QUERY`. For example: + +```sql +BATCH ON test.t2.id LIMIT 1 +DRY RUN QUERY INSERT INTO t +SELECT t2.id, t2.v, t3.id FROM t2, t3 WHERE t2.id = t3.id +``` + +To avoid the error, you can move the condition related to other tables in the `WHERE` clause to the `ON` condition in the `JOIN` clause. For example: + +```sql +BATCH ON test.t2.id LIMIT 1 +INSERT INTO t +SELECT t2.id, t2.v, t3.id FROM t2 JOIN t3 ON t2.id = t3.id +``` + +``` ++----------------+---------------+ +| number of jobs | job status | ++----------------+---------------+ +| 0 | all succeeded | ++----------------+---------------+ +``` + +### The `Unknown column '.' in 'where clause'` error occurs when using table aliases in non-transactional DML statements + +When you execute a non-transactional DML statement, TiDB internally constructs a query for dividing batches and then generates the actual split execution statements. You can use [`DRY RUN QUERY`](/non-transactional-dml.md#query-the-batch-dividing-statement) and [`DRY RUN`](/non-transactional-dml.md#query-the-statements-corresponding-to-the-first-and-the-last-batches) to view these two types of statements, respectively. + +In the current version, the rewritten statements might not preserve the table aliases from the original DML statement. Therefore, if you use the `.` format to reference columns in a `WHERE` clause or other expressions, an `Unknown column` error might occur. + +For example, the following statement might return an error in some cases: + +```sql +BATCH ON t_old.id LIMIT 5000 +INSERT INTO t_new +SELECT * FROM t_old AS t +WHERE t.c1 IS NULL; +``` + +To avoid this error, follow these recommendations: + +- Avoid using table aliases in non-transactional DML statements. For example, rewrite `t.c1` as `c1` or `t_old.c1`. +- When specifying the [shard column](#parameter-description), do not use table aliases. For example, rewrite `BATCH ON t.id` as `BATCH ON db.t_old.id` or `BATCH ON t_old.id`. +- Before execution, use `DRY RUN QUERY` or `DRY RUN` to preview the rewritten statements and verify that they meet your expectations. + ### The actual batch size is not the same as the specified batch size During the execution of a non-transactional DML statement, the size of data to be processed in the last batch might be smaller than the specified batch size. diff --git a/online-unsafe-recovery.md b/online-unsafe-recovery.md index 2310c2b546671..591dc9b36c019 100644 --- a/online-unsafe-recovery.md +++ b/online-unsafe-recovery.md @@ -38,7 +38,7 @@ Before using Online Unsafe Recovery, make sure that the following requirements a ### Step 1. Specify the stores that cannot be recovered -To trigger automatic recovery, use PD Control to execute [`unsafe remove-failed-stores [,,...]`](/pd-control.md#unsafe-remove-failed-stores-store-ids--show) and specify **all** the TiKV nodes that cannot be recovered, seperated by commas. +To trigger automatic recovery, use PD Control to execute [`unsafe remove-failed-stores [,,...]`](/pd-control.md#unsafe-remove-failed-stores-store-ids--show) and specify **all** the TiKV nodes that cannot be recovered, separated by commas. {{< copyable "shell-regular" >}} @@ -174,7 +174,7 @@ After the recovery is completed, the data and index might be inconsistent. Use t ADMIN CHECK TABLE table_name; ``` -If there are inconsistent indexes, you can fix the index inconsistency by renaming the old index, creating a new index, and then droping the old index. +If there are inconsistent indexes, you can fix the index inconsistency by renaming the old index, creating a new index, and then dropping the old index. 1. Rename the old index: diff --git a/optimistic-transaction.md b/optimistic-transaction.md index 147ad9fa41966..0534bb54c627b 100644 --- a/optimistic-transaction.md +++ b/optimistic-transaction.md @@ -1,7 +1,6 @@ --- title: TiDB Optimistic Transaction Model summary: Learn the optimistic transaction model in TiDB. -aliases: ['/docs/dev/optimistic-transaction/','/docs/dev/reference/transactions/transaction-optimistic/','/docs/dev/reference/transactions/transaction-model/'] --- # TiDB Optimistic Transaction Model diff --git a/optimizer-fix-controls.md b/optimizer-fix-controls.md index af1a6093716f3..a918a31d7effb 100644 --- a/optimizer-fix-controls.md +++ b/optimizer-fix-controls.md @@ -14,11 +14,11 @@ Therefore, TiDB provides the Optimizer Fix Controls feature that allows you to m ## Introduction to `tidb_opt_fix_control` -Starting from v7.1.0, TiDB provides the [`tidb_opt_fix_control`](/system-variables.md#tidb_opt_fix_control-new-in-v710) system variable to control the behavior of the optimizer in a more fine-grained way. +Starting from v6.5.3 and v7.1.0, TiDB provides the [`tidb_opt_fix_control`](/system-variables.md#tidb_opt_fix_control-new-in-v653-and-v710) system variable to control the behavior of the optimizer in a more fine-grained way. Each fix is a control item used to adjust the behavior in the TiDB optimizer for one particular purpose. It is denoted by a number that corresponds to a GitHub Issue that contains the technical details of the behavior change. For example, for fix `44262`, you can review what it controls in [Issue 44262](https://github.com/pingcap/tidb/issues/44262). -The [`tidb_opt_fix_control`](/system-variables.md#tidb_opt_fix_control-new-in-v710) system variable accepts multiple fixes as one value, separated by commas (`,`). The format is `"<#issue1>:,<#issue2>:,...,<#issueN>:"`, where `<#issueN>` is the fix number. For example: +The [`tidb_opt_fix_control`](/system-variables.md#tidb_opt_fix_control-new-in-v653-and-v710) system variable accepts multiple fixes as one value, separated by commas (`,`). The format is `"<#issue1>:,<#issue2>:,...,<#issueN>:"`, where `<#issueN>` is the fix number. For example: ```sql SET SESSION tidb_opt_fix_control = '44262:ON,44389:ON'; @@ -26,13 +26,13 @@ SET SESSION tidb_opt_fix_control = '44262:ON,44389:ON'; ## Optimizer Fix Controls reference -### [`44262`](https://github.com/pingcap/tidb/issues/44262) New in v7.2.0 +### [`44262`](https://github.com/pingcap/tidb/issues/44262) New in v6.5.3 and v7.2.0 - Default value: `OFF` - Possible values: `ON`, `OFF` - This variable controls whether to allow the use of [Dynamic pruning mode](/partitioned-table.md#dynamic-pruning-mode) to access the partitioned table when the [GlobalStats](/statistics.md#collect-statistics-of-partitioned-tables-in-dynamic-pruning-mode) are missing. -### [`44389`](https://github.com/pingcap/tidb/issues/44389) New in v7.2.0 +### [`44389`](https://github.com/pingcap/tidb/issues/44389) New in v6.5.3 and v7.2.0 - Default value: `OFF` - Possible values: `ON`, `OFF` @@ -44,13 +44,13 @@ SET SESSION tidb_opt_fix_control = '44262:ON,44389:ON'; - Possible values: `[0, 2147483647]` - To save memory, Plan Cache does not cache queries with parameters exceeding the specified number of this variable. `0` means no limit. -### [`44830`](https://github.com/pingcap/tidb/issues/44830) New in v7.3.0 +### [`44830`](https://github.com/pingcap/tidb/issues/44830) New in v6.5.7 and v7.3.0 - Default value: `OFF` - Possible values: `ON`, `OFF` - This variable controls whether Plan Cache is allowed to cache execution plans with the `PointGet` operator generated during physical optimization. -### [`44855`](https://github.com/pingcap/tidb/issues/44855) New in v7.3.0 +### [`44855`](https://github.com/pingcap/tidb/issues/44855) New in v6.5.4 and v7.3.0 - Default value: `OFF` - Possible values: `ON`, `OFF` @@ -63,4 +63,26 @@ SET SESSION tidb_opt_fix_control = '44262:ON,44389:ON'; - Default value: `1000` - Possible values: `[0, 2147483647]` - This variable sets the threshold for the optimizer's heuristic strategy to select access paths. If the estimated rows for an access path (such as `Index_A`) is much smaller than that of other access paths (default `1000` times), the optimizer skips the cost comparison and directly selects `Index_A`. -- `0` means to disable this heuristic strategy. \ No newline at end of file +- `0` means to disable this heuristic strategy. + +### [`45798`](https://github.com/pingcap/tidb/issues/45798) New in v7.5.0 + +- Default value: `ON` +- Possible values: `ON`, `OFF` +- This variable controls whether Plan Cache is allowed to cache execution plans that access [generated columns](/generated-columns.md). + +### [`46177`](https://github.com/pingcap/tidb/issues/46177) New in v6.5.6, v7.1.3 and v7.5.0 + +- Default value: `OFF` +- Possible values: `ON`, `OFF` +- This variable controls whether the optimizer explores enforced plans during query optimization after finding an unenforced plan. + +### [`56318`](https://github.com/pingcap/tidb/issues/56318) + +> **Note:** +> +> This is only available for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter). + +- Default value: `ON` +- Possible values: `ON`, `OFF` +- This variable controls whether to avoid calculating the heavy expression used in the `ORDER BY` statement twice. \ No newline at end of file diff --git a/optimizer-hints.md b/optimizer-hints.md index d45bde2200a46..314e1684c1cf8 100644 --- a/optimizer-hints.md +++ b/optimizer-hints.md @@ -1,7 +1,6 @@ --- title: Optimizer Hints summary: Use Optimizer Hints to influence query execution plans -aliases: ['/docs/dev/optimizer-hints/','/docs/dev/reference/performance/optimizer-hints/'] --- # Optimizer Hints @@ -139,15 +138,21 @@ SELECT /*+ NO_MERGE_JOIN(t1, t2) */ * FROM t1, t2 WHERE t1.id = t2.id; ### INL_JOIN(t1_name [, tl_name ...]) +> **Note:** +> +> In some cases, the `INL_JOIN` hint might not take effect. For more information, see [`INL_JOIN` hint does not take effect](#inl_join-hint-does-not-take-effect). + The `INL_JOIN(t1_name [, tl_name ...])` hint tells the optimizer to use the index nested loop join algorithm for the given table(s). This algorithm might consume less system resources and take shorter processing time in some scenarios and might produce an opposite result in other scenarios. If the result set is less than 10,000 rows after the outer table is filtered by the `WHERE` condition, it is recommended to use this hint. For example: {{< copyable "sql" >}} ```sql -select /*+ INL_JOIN(t1, t2) */ * from t1, t2 where t1.id = t2.id; +SELECT /*+ INL_JOIN(t1, t2) */ * FROM t1, t2, t3 WHERE t1.id = t2.id AND t2.id = t3.id; ``` -The parameter(s) given in `INL_JOIN()` is the candidate table for the inner table when you create the query plan. For example, `INL_JOIN(t1)` means that TiDB only considers using `t1` as the inner table to create a query plan. If the candidate table has an alias, you must use the alias as the parameter in `INL_JOIN()`; if it does not has an alias, use the table's original name as the parameter. For example, in the `select /*+ INL_JOIN(t1) */ * from t t1, t t2 where t1.a = t2.b;` query, you must use the `t` table's alias `t1` or `t2` rather than `t` as `INL_JOIN()`'s parameter. +In the preceding SQL statement, the `INL_JOIN(t1, t2)` hint tells the optimizer to use the index nested loop join algorithm for `t1` and `t2`. Note that this does not mean that the index nested loop join algorithm is used between `t1` and `t2`. Instead, the hint indicates that `t1` and `t2` each use the index nested loop join algorithm with another table (`t3`). + +The parameter(s) given in `INL_JOIN()` is the candidate table for the inner table when you create the query plan. For example, `INL_JOIN(t1)` means that TiDB only considers using `t1` as the inner table to create a query plan. If the candidate table has an alias, you must use the alias as the parameter in `INL_JOIN()`; if it does not have an alias, use the table's original name as the parameter. For example, in the `select /*+ INL_JOIN(t1) */ * from t t1, t t2 where t1.a = t2.b;` query, you must use the `t` table's alias `t1` or `t2` rather than `t` as `INL_JOIN()`'s parameter. > **Note:** > @@ -571,9 +576,8 @@ The `LEADING` hint does not take effect in the following situations: + The optimizer cannot perform join operations according to the order as specified by the `LEADING` hint. + The `straight_join()` hint already exists. + The query contains an outer join together with the Cartesian product. -+ Any of the `MERGE_JOIN`, `INL_JOIN`, `INL_HASH_JOIN`, and `HASH_JOIN` hints is used at the same time. -In the above situations, a warning is generated. +In the preceding situations, a warning is generated. ```sql -- Multiple `LEADING` hints are specified. @@ -929,9 +933,55 @@ The warning is as follows: In this case, you need to place the hint directly after the `SELECT` keyword. For more details, see the [Syntax](#syntax) section. -### INL_JOIN hint does not take effect due to collation incompatibility +### `INL_JOIN` hint does not take effect + +#### `INL_JOIN` hint does not take effect when built-in functions are used on columns for joining tables + +In some cases, if you use a built-in function on a column that joins tables, the optimizer might fail to choose the `IndexJoin` plan, resulting in the `INL_JOIN` hint not taking effect either. + +For example, the following query uses the built-in function `substr` on the column `tname` that joins tables: + +```sql +CREATE TABLE t1 (id varchar(10) primary key, tname varchar(10)); +CREATE TABLE t2 (id varchar(10) primary key, tname varchar(10)); +EXPLAIN SELECT /*+ INL_JOIN(t1, t2) */ * FROM t1, t2 WHERE t1.id=t2.id and SUBSTR(t1.tname,1,2)=SUBSTR(t2.tname,1,2); +``` + +The execution plan is as follows: + +```sql ++------------------------------+----------+-----------+---------------+-----------------------------------------------------------------------+ +| id | estRows | task | access object | operator info | ++------------------------------+----------+-----------+---------------+-----------------------------------------------------------------------+ +| HashJoin_12 | 12500.00 | root | | inner join, equal:[eq(test.t1.id, test.t2.id) eq(Column#5, Column#6)] | +| ├─Projection_17(Build) | 10000.00 | root | | test.t2.id, test.t2.tname, substr(test.t2.tname, 1, 2)->Column#6 | +| │ └─TableReader_19 | 10000.00 | root | | data:TableFullScan_18 | +| │ └─TableFullScan_18 | 10000.00 | cop[tikv] | table:t2 | keep order:false, stats:pseudo | +| └─Projection_14(Probe) | 10000.00 | root | | test.t1.id, test.t1.tname, substr(test.t1.tname, 1, 2)->Column#5 | +| └─TableReader_16 | 10000.00 | root | | data:TableFullScan_15 | +| └─TableFullScan_15 | 10000.00 | cop[tikv] | table:t1 | keep order:false, stats:pseudo | ++------------------------------+----------+-----------+---------------+-----------------------------------------------------------------------+ +7 rows in set, 1 warning (0.01 sec) +``` + +```sql +SHOW WARNINGS; +``` + +``` ++---------+------+------------------------------------------------------------------------------------+ +| Level | Code | Message | ++---------+------+------------------------------------------------------------------------------------+ +| Warning | 1815 | Optimizer Hint /*+ INL_JOIN(t1, t2) */ or /*+ TIDB_INLJ(t1, t2) */ is inapplicable | ++---------+------+------------------------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +As you can see from the preceding example, the `INL_JOIN` hint does not take effect. This is due to a limitation of the optimizer that prevents using the `Projection` or `Selection` operator as the probe side of `IndexJoin`. + +#### `INL_JOIN`, `INL_HASH_JOIN`, and `INL_MERGE_JOIN` hints do not take effect due to collation incompatibility -When the collation of the join key is incompatible between two tables, the `IndexJoin` operator cannot be utilized to execute the query. In this case, the [`INL_JOIN` hint](#inl_joint1_name--tl_name-) does not take effect. For example: +When the collation of the join key is incompatible between two tables, the `IndexJoin` operator cannot be utilized to execute the query. In this case, the [`INL_JOIN`](#inl_joint1_name--tl_name-), [`INL_HASH_JOIN`](#inl_hash_join), and [`INL_MERGE_JOIN`](#inl_merge_join) hints do not take effect. For example: ```sql CREATE TABLE t1 (k varchar(8), key(k)) COLLATE=utf8mb4_general_ci; @@ -966,9 +1016,9 @@ SHOW WARNINGS; 1 row in set (0.00 sec) ``` -### `INL_JOIN` hint does not take effect because of join order +#### `INL_JOIN` hint does not take effect due to join order -The [`INL_JOIN(t1, t2)`](#inl_joint1_name--tl_name-) or `TIDB_INLJ(t1, t2)` hint semantically instructs `t1` and `t2` to act as inner tables in an `IndexJoin` operator to join with other tables, rather than directly joining them using an `IndexJoin`operator. For example: +The [`INL_JOIN(t1, t2)`](#inl_joint1_name--tl_name-) or `TIDB_INLJ(t1, t2)` hint semantically instructs `t1` and `t2` to act as inner tables in an `IndexJoin` operator to join with other tables, rather than directly joining them using an `IndexJoin` operator. For example: ```sql EXPLAIN SELECT /*+ inl_join(t1, t3) */ * FROM t1, t2, t3 WHERE t1.id = t2.id AND t2.id = t3.id AND t1.id = t3.id; @@ -1022,7 +1072,7 @@ EXPLAIN SELECT /*+ NO_HASH_JOIN(t1), NO_MERGE_JOIN(t1) */ * FROM t1, t2 WHERE t1 ERROR 1815 (HY000): Internal : Can't find a proper physical plan for this query ``` -- The system variable [`tidb_opt_enable_hash_join`](/system-variables.md#tidb_opt_enable_hash_join-new-in-v740) is set to `OFF`, and all other join types are also excluded. +- The system variable [`tidb_opt_enable_hash_join`](/system-variables.md#tidb_opt_enable_hash_join-new-in-v656-v712-and-v740) is set to `OFF`, and all other join types are also excluded. ```sql CREATE TABLE t1 (a INT); diff --git a/oracle-functions-to-tidb.md b/oracle-functions-to-tidb.md index a065ce85a44cd..44472b49c9640 100644 --- a/oracle-functions-to-tidb.md +++ b/oracle-functions-to-tidb.md @@ -33,7 +33,7 @@ The following table shows the comparisons between some Oracle and TiDB functions | Get a random sequence value | `SYS_GUID()` | `UUID()` | TiDB returns a Universal Unique Identifier (UUID). | | Left join or right join | `SELECT * FROM a, b WHERE a.id = b.id(+);`
`SELECT * FROM a, b WHERE a.id(+) = b.id;` | `SELECT * FROM a LEFT JOIN b ON a.id = b.id;`
`SELECT * FROM a RIGHT JOIN b ON a.id = b.id;` | In a correlated query, TiDB does not support using (+) to left join or right join. You can use `LEFT JOIN` or `RIGHT JOIN` instead. | | `NVL()` | `NVL(key,val)` | `IFNULL(key,val)` | If the value of the field is `NULL`, it returns `val`; otherwise, it returns the value of the field. | -| `NVL2()` | `NVL2(key, val1, val2)` | `IF(key is NULL, val1, val2)` | If the value of the field is not `NULL`, it returns `val1`; otherwise, it returns `val2`. | +| `NVL2()` | `NVL2(key, val1, val2)` | `IF(key is NOT NULL, val1, val2)` | If the value of the field is not `NULL`, it returns `val1`; otherwise, it returns `val2`. | | `DECODE()` |
  • `DECODE(key,val1,val2,val3)`
  • `DECODE(value,if1,val1,if2,val2,...,ifn,valn,val)`
  • |
  • `IF(key=val1,val2,val3)`
  • `CASE WHEN value=if1 THEN val1 WHEN value=if2 THEN val2,...,WHEN value=ifn THEN valn ELSE val END`
  • |
  • If the value of the field is `val1`, then it returns `val2`; otherwise it returns `val3`.
  • When the value of the field satisfies condition 1 (`if1`), it returns `val1`. When it satisfies condition 2 (`if2`), it returns `val2`. When it satisfies condition 3 (`if3`), it returns `val3`.
  • | | Concatenate the string `a` and `b` | 'a' \|\| 'b' | `CONCAT('a','b')` | | | Get the length of a string | `LENGTH(str)` | `CHAR_LENGTH(str)` | | @@ -65,13 +65,13 @@ TiDB distinguishes between `NULL` and an empty string `''`. Oracle supports reading and writing to the same table in an `INSERT` statement. For example: ```sql -INSERT INTO table1 VALUES (feild1,(SELECT feild2 FROM table1 WHERE...)) +INSERT INTO table1 VALUES (field1,(SELECT field2 FROM table1 WHERE...)) ``` TiDB does not support reading and writing to the same table in a `INSERT` statement. For example: ```sql -INSERT INTO table1 VALUES (feild1,(SELECT T.fields2 FROM table1 T WHERE...)) +INSERT INTO table1 VALUES (field1,(SELECT T.fields2 FROM table1 T WHERE...)) ``` ### Get the first n rows from a query diff --git a/overview.md b/overview.md index 65defc64836e4..f316167f261e8 100644 --- a/overview.md +++ b/overview.md @@ -1,10 +1,9 @@ --- -title: TiDB Introduction +title: What is TiDB Self-Managed summary: Learn about the key features and usage scenarios of TiDB. -aliases: ['/docs/dev/key-features/','/tidb/dev/key-features','/docs/dev/overview/'] --- -# TiDB Introduction +# What is TiDB Self-Managed -The PD configuration file supports more options than command-line parameters. You can find the default configuration file [here](https://github.com/pingcap/pd/blob/master/conf/config.toml). +The PD configuration file supports more options than command-line parameters. You can find the default configuration file [here](https://github.com/pingcap/pd/blob/release-7.5/conf/config.toml). This document only describes parameters that are not included in command-line parameters. Check [here](/command-line-flags-for-pd-configuration.md) for the command line parameters. @@ -258,6 +257,13 @@ Configuration items related to monitoring Configuration items related to scheduling +> **Note:** +> +> To modify these PD configuration items related to `schedule`, choose one of the following methods based on your cluster status: +> +> - For clusters to be newly deployed, you can modify the PD configuration file directly. +> - For existing clusters, use the command-line tool [PD Control](/pd-control.md) to make changes instead. Direct modifications to these PD configuration items related to `schedule` in the configuration file do not take effect on existing clusters. + ### `max-merge-region-size` + Controls the size limit of `Region Merge`. When the Region size is greater than the specified value, PD does not merge the Region with the adjacent Regions. @@ -279,6 +285,12 @@ Configuration items related to scheduling + Controls the time interval between the `split` and `merge` operations on the same Region. That means a newly split Region will not be merged for a while. + Default value: `1h` +### `max-movable-hot-peer-size` New in v6.1.0 + ++ Controls the maximum Region size that can be scheduled for hot Region scheduling. ++ Default value: `512` ++ Unit: MiB + ### `max-snapshot-count` + Controls the maximum number of snapshots that a single store receives or sends at the same time. PD schedulers depend on this configuration to prevent the resources used for normal traffic from being preempted. @@ -432,16 +444,20 @@ Configuration items related to replicas + Default value: `true` + See [Placement Rules](/configure-placement-rules.md). -## `label-property` +## `label-property` (deprecated) + +Configuration items related to labels, which only support the `reject-leader` type. -Configuration items related to labels +> **Note:** +> +> Starting from v5.2, the configuration items related to labels are deprecated. It is recommended to use [Placement Rules](/configure-placement-rules.md#scenario-2-place-five-replicas-in-three-data-centers-in-the-proportion-of-221-and-the-leader-should-not-be-in-the-third-data-center) to configure the replica policy. -### `key` +### `key` (deprecated) + The label key for the store that rejected the Leader + Default value: `""` -### `value` +### `value` (deprecated) + The label value for the store that rejected the Leader + Default value: `""` @@ -481,7 +497,7 @@ Configuration items related to the [TiDB Dashboard](/dashboard/dashboard-intro.m Configuration items related to the replication mode of all Regions. See [Enable the DR Auto-Sync mode](/two-data-centers-in-one-city-deployment.md#enable-the-dr-auto-sync-mode) for details. -## Controllor +## controller This section describes the configuration items that are built into PD for [Resource Control](/tidb-resource-control.md). @@ -498,7 +514,7 @@ The following are the configuration items about the [Request Unit (RU)](/tidb-re #### `read-base-cost` + Basis factor for conversion from a read request to RU -+ Default value: 0.25 ++ Default value: 0.125 #### `write-base-cost` diff --git a/pd-control.md b/pd-control.md index 4b9f51c2af157..741b34c09ca34 100644 --- a/pd-control.md +++ b/pd-control.md @@ -1,7 +1,6 @@ --- title: PD Control User Guide summary: Use PD Control to obtain the state information of a cluster and tune a cluster. -aliases: ['/docs/dev/pd-control/','/docs/dev/reference/tools/pd-control/'] --- # PD Control User Guide @@ -24,12 +23,12 @@ To obtain `pd-ctl` of the latest version, download the TiDB server installation | Installation package | OS | Architecture | SHA256 checksum | | :------------------------------------------------------------------------ | :------- | :---- | :--------------------------------------------------------------- | -| `https://download.pingcap.org/tidb-community-server-{version}-linux-amd64.tar.gz` (pd-ctl) | Linux | amd64 | `https://download.pingcap.org/tidb-community-server-{version}-linux-amd64.tar.gz.sha256` | -| `https://download.pingcap.org/tidb-community-server-{version}-linux-arm64.tar.gz` (pd-ctl) | Linux | arm64 | `https://download.pingcap.org/tidb-community-server-{version}-linux-arm64.tar.gz.sha256` | +| `https://download.pingcap.com/tidb-community-server-{version}-linux-amd64.tar.gz` (pd-ctl) | Linux | amd64 | `https://download.pingcap.com/tidb-community-server-{version}-linux-amd64.tar.gz.sha256` | +| `https://download.pingcap.com/tidb-community-server-{version}-linux-arm64.tar.gz` (pd-ctl) | Linux | arm64 | `https://download.pingcap.com/tidb-community-server-{version}-linux-arm64.tar.gz.sha256` | > **Note:** > -> `{version}` in the link indicates the version number of TiDB. For example, the download link for `v7.4.0` in the `amd64` architecture is `https://download.pingcap.org/tidb-community-server-v7.4.0-linux-amd64.tar.gz`. +> `{version}` in the link indicates the version number of TiDB. For example, the download link for `v{{{ .tidb-version }}}` in the `amd64` architecture is `https://download.pingcap.com/tidb-community-server-v{{{ .tidb-version }}}-linux-amd64.tar.gz`. ### Compile from source code @@ -172,7 +171,7 @@ Usage: } >> config show cluster-version // Display the current version of the cluster, which is the current minimum version of TiKV nodes in the cluster and does not correspond to the binary version. -"5.2.2" +"{{{ .tidb-version }}}" ``` - `max-snapshot-count` controls the maximum number of snapshots that a single store receives or sends out at the same time. The scheduler is restricted by this configuration to avoid taking up normal application resources. When you need to improve the speed of adding replicas or balancing, increase this value. @@ -308,7 +307,7 @@ Usage: - `cluster-version` is the version of the cluster, which is used to enable or disable some features and to deal with the compatibility issues. By default, it is the minimum version of all normally running TiKV nodes in the cluster. You can set it manually only when you need to roll it back to an earlier version. ```bash - config set cluster-version 1.0.8 // Set the version of the cluster to 1.0.8 + config set cluster-version {{{ .tidb-version }}} // Set the version of the cluster to {{{ .tidb-version }}} ``` - `replication-mode` controls the replication mode of Regions in the dual data center scenario. See [Enable the DR Auto-Sync mode](/two-data-centers-in-one-city-deployment.md#enable-the-dr-auto-sync-mode) for details. @@ -333,7 +332,7 @@ Usage: - `store-limit-mode` is used to control the mode of limiting the store speed. The optional modes are `auto` and `manual`. In `auto` mode, the stores are automatically balanced according to the load (deprecated). -- `store-limit-version` controls the version of the store limit formula. In v1 mode, you can manually modify the `store limit` to limit the scheduling speed of a single TiKV. The v2 mode is an experimental feature. In v2 mode, you do not need to manually set the `store limit` value, as PD dynamically adjusts it based on the capability of TiKV snapshots. For more details, refer to [Principles of store limit v2](/configure-store-limit.md#principles-of-store-limit-v2). +- `store-limit-version` controls the version of the store limit formula. In v1 mode, you can manually modify the `store limit` to limit the scheduling speed of a single TiKV. The v2 mode is an experimental feature. It is not recommended that you use it in the production environment. In v2 mode, you do not need to manually set the `store limit` value, as PD dynamically adjusts it based on the capability of TiKV snapshots. For more details, refer to [Principles of store limit v2](/configure-store-limit.md#principles-of-store-limit-v2). ```bash config set store-limit-version v2 // using store limit v2 @@ -442,6 +441,10 @@ Usage: ### `member [delete | leader_priority | leader [show | resign | transfer ]]` +> **Note:** +> +> **DO NOT** use the `member delete` command to remove PD nodes in a production environment. To remove a PD node, see [Scale in a TiDB/PD/TiKV cluster](/scale-tidb-using-tiup.md#scale-in-a-tidbpdtikv-cluster) and [Manually Scale TiDB on Kubernetes](https://docs.pingcap.com/tidb-in-kubernetes/stable/scale-a-tidb-cluster/). + Use this command to view the PD members, remove a specified member, or configure the priority of leader. Usage: @@ -757,6 +760,42 @@ Usage: } ``` +### `resource-manager [command]` + +#### View the controller configuration of Resource Control + +```bash +resource-manager config controller show +``` + +```bash +{ + "degraded-mode-wait-duration": "0s", + "ltb-max-wait-duration": "30s", + "request-unit": { # Configurations of RU. Do not modify. + "read-base-cost": 0.125, + "read-per-batch-base-cost": 0.5, + "read-cost-per-byte": 0.0000152587890625, + "write-base-cost": 1, + "write-per-batch-base-cost": 1, + "write-cost-per-byte": 0.0009765625, + "read-cpu-ms-cost": 0.3333333333333333 + }, + "enable-controller-trace-log": "false" +} +``` + +- `ltb-max-wait-duration`: the maximum waiting time of Local Token Bucket (LTB). The default value is `30s`, and the value range is `[0, 24h]`. If the estimated [Request Unit (RU)](/tidb-resource-control.md#what-is-request-unit-ru) consumption of the SQL request exceeds the current accumulated RU of LTB, the request needs to wait for a certain period of time. If the estimated waiting time exceeds this maximum value, an error message [`ERROR 8252 (HY000) : Exceeded resource group quota limitation`](/error-codes.md) is returned to the application in advance. Increasing this value can reduce the occurrence of encountering `ERROR 8252` in cases of sudden concurrency increase, large transactions, and large queries. +- `enable-controller-trace-log`: control whether to enable the controller diagnostic log. + +#### Modify the controller configuration of Resource Control + +To modify the `ltb-max-wait-duration` configuration, use the following command: + +```bash +pd-ctl resource-manager config controller set ltb-max-wait-duration 30m +``` + ### `scheduler [show | add | remove | pause | resume | config | describe]` Use this command to view and control the scheduling policy. @@ -764,20 +803,20 @@ Use this command to view and control the scheduling policy. Usage: ```bash ->> scheduler show // Display all created schedulers ->> scheduler add grant-leader-scheduler 1 // Schedule all the leaders of the Regions on store 1 to store 1 ->> scheduler add evict-leader-scheduler 1 // Move all the Region leaders on store 1 out ->> scheduler config evict-leader-scheduler // Display the stores in which the scheduler is located since v4.0.0 ->> scheduler add shuffle-leader-scheduler // Randomly exchange the leader on different stores ->> scheduler add shuffle-region-scheduler // Randomly scheduling the Regions on different stores ->> scheduler add evict-slow-store-scheduler // When there is one and only one slow store, evict all Region leaders of that store ->> scheduler remove grant-leader-scheduler-1 // Remove the corresponding scheduler, and `-1` corresponds to the store ID ->> scheduler pause balance-region-scheduler 10 // Pause the balance-region scheduler for 10 seconds ->> scheduler pause all 10 // Pause all schedulers for 10 seconds ->> scheduler resume balance-region-scheduler // Continue to run the balance-region scheduler ->> scheduler resume all // Continue to run all schedulers ->> scheduler config balance-hot-region-scheduler // Display the configuration of the balance-hot-region scheduler ->> scheduler describe balance-region-scheduler // Display the running state and related diagnostic information of the balance-region scheduler +>> scheduler show // Display all created schedulers +>> scheduler add grant-leader-scheduler 1 // Schedule all the leaders of the Regions on store 1 to store 1 +>> scheduler add evict-leader-scheduler 1 // Move all the Region leaders on store 1 out +>> scheduler config evict-leader-scheduler // Display the stores in which the scheduler is located since v4.0.0 +>> scheduler config evict-leader-scheduler add-store 2 // Add leader eviction scheduling for store 2 +>> scheduler config evict-leader-scheduler delete-store 2 // Remove leader eviction scheduling for store 2 +>> scheduler add evict-slow-store-scheduler // When there is one and only one slow store, evict all Region leaders of that store +>> scheduler remove grant-leader-scheduler-1 // Remove the corresponding scheduler, and `-1` corresponds to the store ID +>> scheduler pause balance-region-scheduler 10 // Pause the balance-region scheduler for 10 seconds +>> scheduler pause all 10 // Pause all schedulers for 10 seconds +>> scheduler resume balance-region-scheduler // Continue to run the balance-region scheduler +>> scheduler resume all // Continue to run all schedulers +>> scheduler config balance-hot-region-scheduler // Display the configuration of the balance-hot-region scheduler +>> scheduler describe balance-region-scheduler // Display the running state and related diagnostic information of the balance-region scheduler ``` ### `scheduler describe balance-region-scheduler` @@ -923,6 +962,24 @@ Usage: scheduler config balance-hot-region-scheduler set enable-for-tiflash true ``` +### `scheduler config evict-leader-scheduler` + +Use this command to view and manage the configuration of the `evict-leader-scheduler`. + +- When an `evict-leader-scheduler` already exists, use the `add-store` subcommand to add leader eviction scheduling for the specified store: + + ```bash + scheduler config evict-leader-scheduler add-store 2 // Add leader eviction scheduling for store 2 + ``` + +- When an `evict-leader-scheduler` already exists, use the `delete-store` subcommand to remove leader eviction scheduling for the specified store: + + ```bash + scheduler config evict-leader-scheduler delete-store 2 // Remove leader eviction scheduling for store 2 + ``` + + If all store configurations of an `evict-leader-scheduler` are removed, the scheduler itself is automatically removed. + ### `service-gc-safepoint` Use this command to query the current GC safepoint and service GC safepoint. The output is as follows: diff --git a/pd-recover.md b/pd-recover.md index 24b3bd3049d80..7fbb190a482ee 100644 --- a/pd-recover.md +++ b/pd-recover.md @@ -1,7 +1,6 @@ --- title: PD Recover User Guide summary: Use PD Recover to recover a PD cluster which cannot start or provide services normally. -aliases: ['/docs/dev/pd-recover/','/docs/dev/reference/tools/pd-recover/'] --- # PD Recover User Guide diff --git a/performance-schema/performance-schema-session-connect-attrs.md b/performance-schema/performance-schema-session-connect-attrs.md index e1a758d7d2b33..dd0c73b65bee3 100644 --- a/performance-schema/performance-schema-session-connect-attrs.md +++ b/performance-schema/performance-schema-session-connect-attrs.md @@ -52,7 +52,7 @@ TABLE SESSION_CONNECT_ATTRS; | PROCESSLIST_ID | ATTR_NAME | ATTR_VALUE | ORDINAL_POSITION | +----------------+-----------------+------------+------------------+ | 2097154 | _client_name | libmysql | 0 | -| 2097154 | _client_version | 8.1.0 | 1 | +| 2097154 | _client_version | {{{ .tidb-version }}} | 1 | | 2097154 | _os | Linux | 2 | | 2097154 | _pid | 1299203 | 3 | | 2097154 | _platform | x86_64 | 4 | diff --git a/performance-tuning-methods.md b/performance-tuning-methods.md index 9d64deb6165c1..f3941af5322e2 100644 --- a/performance-tuning-methods.md +++ b/performance-tuning-methods.md @@ -187,7 +187,7 @@ The number of `StmtPrepare` commands per second is much greater than that of `St ![OLTP-Query](/media/performance/prepared_statement_leaking.png) -- In the QPS panel, the red bold line indicates the number of failed queries, and the Y axis on the right indicates the coordinate value of the number. In this example, the number of failed quries per second is 74.6. +- In the QPS panel, the red bold line indicates the number of failed queries, and the Y axis on the right indicates the coordinate value of the number. In this example, the number of failed queries per second is 74.6. - In the CPS By Type panel, the number of `StmtPrepare` commands per second is much greater than that of `StmtClose` per second, which indicates that an object leak occurs in the application for prepared statements. - In the Queries Using Plan Cache OPS panel, `avg-miss` is almost equal to `StmtExecute` in the CPS By Type panel, which indicates that almost all SQL executions miss the execution plan cache. @@ -256,7 +256,7 @@ The Duration panel contains the average and P99 latency of all statements, and t - in-txn: The interval between processing the previous SQL and receiving the next SQL statement when the connection is within a transaction. - not-in-txn: The interval between processing the previous SQL and receiving the next SQL statement when the connection is not within a transaction. -An applications perform transactions with the same database connction. By comparing the average query latency with the connection idle duration, you can determine if TiDB is the bottleneck for overall system, or if user response time jitter is caused by TiDB. +An applications perform transactions with the same database connection. By comparing the average query latency with the connection idle duration, you can determine if TiDB is the bottleneck for overall system, or if user response time jitter is caused by TiDB. - If the application workload is not read-only and contains transactions, by comparing the average query latency with `avg-in-txn`, you can determine the proportion in processing transactions inside and outside the database, and identify the bottleneck in user response time. - If the application workload is read-only or autocommit mode is on, you can compare the average query latency with `avg-not-in-txn`. @@ -489,6 +489,6 @@ For the `Store` thread, `Commit Log Duration` is obviously higher than `Apply Lo ## If my TiDB version is earlier than v6.1.0, what should I do to use the Performance Overview dashboard? -Starting from v6.1.0, Grafana has a built-in Performance Overview dashboard by default. This dashboard is compatible with TiDB v4.x and v5.x versions. If your TiDB is earlier than v6.1.0, you need to manually import [`performance_overview.json`](https://github.com/pingcap/tidb/blob/master/pkg/metrics/grafana/performance_overview.json), as shown in the following figure: +Starting from v6.1.0, Grafana has a built-in Performance Overview dashboard by default. This dashboard is compatible with TiDB v4.x and v5.x versions. If your TiDB is earlier than v6.1.0, you need to manually import [`performance_overview.json`](https://github.com/pingcap/tidb/blob/release-7.5/pkg/metrics/grafana/performance_overview.json), as shown in the following figure: ![Store](/media/performance/import_dashboard.png) diff --git a/performance-tuning-overview.md b/performance-tuning-overview.md index c4ba353e763f3..4332f9cc9b590 100644 --- a/performance-tuning-overview.md +++ b/performance-tuning-overview.md @@ -107,7 +107,7 @@ After identifying the bottleneck of a system through performance analysis, you c According to [Amdahl's Law](https://en.wikipedia.org/wiki/Amdahl%27s_law), the maximum gain from performance tuning depends on the percentage of the optimized part in the overall system. Therefore, you need to identify the system bottlenecks and the corresponding percentage based on the performance data, and then predict the gains after the bottleneck is resolved or optimized. -Note that even if a solution can bring the greatest potential benefits by tunning the largest bottleneck, you still need to evaluate the risks and costs of this solution. For example: +Note that even if a solution can bring the greatest potential benefits by tuning the largest bottleneck, you still need to evaluate the risks and costs of this solution. For example: - The most straightforward tuning objective solution for a resource-overloaded system is to expand its capacity, but in practice, the expansion solution might be too costly to be adopted. - When a slow query in a business module causes a slow response of the entire module, upgrading to a new version of the database can solve the slow query issue, but it might also affect modules that did not have this issue. Therefore, this solution might have a potentially high risk. A low-risk solution is to skip the database version upgrade and rewrite the existing slow queries for the current database version. diff --git a/performance-tuning-practices.md b/performance-tuning-practices.md index 13a52c0db9730..7d31749bf1106 100644 --- a/performance-tuning-practices.md +++ b/performance-tuning-practices.md @@ -13,7 +13,7 @@ This document describes how to use these features together to analyze and compar > > [Top SQL](/dashboard/top-sql.md) and [Continuous Profiling](/dashboard/continuous-profiling.md) are not enabled by default. You need to enable them in advance. -By running the same application with different JDBC configurations in these scenarios, this document shows you how the overall system performance is affected by different interactions between applications and databases, so that you can apply [Best Practices for Developing Java Applications with TiDB](/best-practices/java-app-best-practices.md) for better performance. +By running the same application with different JDBC configurations in these scenarios, this document shows you how the overall system performance is affected by different interactions between applications and databases, so that you can apply [Best Practices for Developing Java Applications with TiDB](/develop/java-app-best-practices.md) for better performance. ## Environment description @@ -413,13 +413,13 @@ The following table lists the performance of seven different scenarios. | Metrics | Scenario 1 | Scenario 2 | Scenario 3 | Scenario 4 | Scenario 5 | Scenario 6 | Scenario 7 | Comparing Scenario 5 with Scenario 2 (%) | Comparing Scenario 7 with Scenario 3 (%) | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| query duration | 479μs | 1120μs | 528μs | 426μs |690μs | 533μs | 313μs | -38% | -51% | +| query duration | 479μs | 1120μs | 528μs | 426μs |690μs | 533μs | 313μs | -38% | -41% | | QPS | 56.3k | 24.2k | 19.7k | 22.1k | 30.9k | 34.9k | 40.9k | +28% | +108% | In these scenarios, Scenario 2 is a common scenario where applications use the Query interface, and Scenario 5 is an ideal scenario where applications use the Prepared Statement interface. -- Comparing Scenario 2 with Scenario 5, you can see that by using best practices for Java application development and caching Prepared Statement objects on the client side, each SQL statement requires only one command and database interaction to hit the execution plan cache, which results in a 38% drop in query latency and a 28% increase in QPS, while the average TiDB CPU utilization drops from 936% to 577%. -- Comparing Scenario 2 with Scenario 7, you can see that with the latest TiDB optimization features such as RC Read and small table cache on top of Scenario 5, latency is reduced by 51% and QPS is increased by 108%, while the average TiDB CPU utilization drops from 936% to 478%. +- Comparing Scenario 5 with Scenario 2, you can see that by using best practices for Java application development and caching Prepared Statement objects on the client side, each SQL statement requires only one command and database interaction to hit the execution plan cache, which results in a 38% drop in query latency and a 28% increase in QPS, while the average TiDB CPU utilization drops from 936% to 577%. +- Comparing Scenario 7 with Scenario 3, you can see that with the latest TiDB optimization features such as RC Read and small table cache on top of Scenario 5, latency is reduced by 41% and QPS is increased by 108%, while the average TiDB CPU utilization drops from 936% to 478%. By comparing the performance of each scenario, we can draw the following conclusions: @@ -427,7 +427,7 @@ By comparing the performance of each scenario, we can draw the following conclus - TiDB is compatible with different commands of the MySQL protocol. When using the Prepared Statement interface and setting the following JDBC connection parameters, the application can achieve its best performance: ``` - useServerPrepStmts=true&cachePrepStmts=true&prepStmtCacheSize=1000&prepStmtCacheSqlLimit=20480&useConfigs= maxPerformance + useServerPrepStmts=true&cachePrepStmts=true&prepStmtCacheSize=1000&prepStmtCacheSqlLimit=20480&useConfigs=maxPerformance ``` - It is recommended that you use TiDB Dashboard (for example, the Top SQL feature and Continuous Profiling feature) and Performance Overview dashboard for performance analysis and tuning. diff --git a/pessimistic-transaction.md b/pessimistic-transaction.md index b759d2af95854..fb25b2b5522c1 100644 --- a/pessimistic-transaction.md +++ b/pessimistic-transaction.md @@ -1,7 +1,6 @@ --- title: TiDB Pessimistic Transaction Mode summary: Learn the pessimistic transaction mode in TiDB. -aliases: ['/docs/dev/pessimistic-transaction/','/docs/dev/reference/transactions/transaction-pessimistic/'] --- # TiDB Pessimistic Transaction Mode diff --git a/placement-rules-in-sql.md b/placement-rules-in-sql.md index 2e35f8828ffc2..73fd997a093f5 100644 --- a/placement-rules-in-sql.md +++ b/placement-rules-in-sql.md @@ -15,7 +15,7 @@ This feature can fulfill the following use cases: > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ## Overview @@ -30,7 +30,7 @@ With the Placement Rules in SQL feature, you can [create placement policies](#cr > **Tip:** > -> The implementation of *Placement Rules in SQL* relies on the *placement rules feature* of PD. For details, refer to [Configure Placement Rules](https://docs.pingcap.com/zh/tidb/stable/configure-placement-rules). In the context of Placement Rules in SQL, *placement rules* might refer to *placement policies* attached to other objects, or to rules that are sent from TiDB to PD. +> The implementation of *Placement Rules in SQL* relies on the *placement rules feature* of PD. For details, refer to [Configure Placement Rules](https://docs.pingcap.com/tidb/stable/configure-placement-rules). In the context of Placement Rules in SQL, *placement rules* might refer to *placement policies* attached to other objects, or to rules that are sent from TiDB to PD. ## Limitations @@ -44,7 +44,7 @@ Placement policies rely on the configuration of labels on TiKV nodes. For exampl -When you create a placement policy, TiDB does not check whether the labels specified in the policy exist. Instead, TiDB performs the check when you attach the policy. Therefore, before attaching a placement policy, make sure that each TiKV node is configured with correct labels. The configuration method for a TiDB Self-Hosted cluster is as follows: +When you create a placement policy, TiDB does not check whether the labels specified in the policy exist. Instead, TiDB performs the check when you attach the policy. Therefore, before attaching a placement policy, make sure that each TiKV node is configured with correct labels. The configuration method for a TiDB Self-Managed cluster is as follows: ``` tikv-server --labels region=,zone=,host= @@ -60,13 +60,13 @@ For detailed configuration methods, see the following examples: > **Note:** > -> For TiDB Dedicated clusters, you can skip these label configuration steps because the labels on TiKV nodes in TiDB Dedicated clusters are configured automatically. +> For TiDB Cloud Dedicated clusters, you can skip these label configuration steps because the labels on TiKV nodes in TiDB Cloud Dedicated clusters are configured automatically. -For TiDB Dedicated clusters, labels on TiKV nodes are configured automatically. +For TiDB Cloud Dedicated clusters, labels on TiKV nodes are configured automatically. @@ -231,8 +231,8 @@ You can configure `CONSTRAINTS`, `FOLLOWER_CONSTRAINTS`, and `LEARNER_CONSTRAINT | CONSTRAINTS format | Description | |----------------------------|-----------------------------------------------------------------------------------------------------------| -| List format | If a constraint to be specified applies to all replicas, you can use a key-value list format. Each key starts with `+` or `-`. For example:
    • `[+region=us-east-1]` means placing data on nodes that have a `region` label as `us-east-1`.
    • `[+region=us-east-1,-type=fault]` means placing data on nodes that have a `region` label as `us-east-1` but do not have a `type` label as `fault`.

    | -| Dictionary format | If you need to specify different numbers of replicas for different constraints, you can use the dictionary format. For example:
    • `FOLLOWER_CONSTRAINTS="{+region=us-east-1: 1,+region=us-east-2: 1,+region=us-west-1: 1}";` means placing one Follower in `us-east-1`, one Follower in `us-east-2`, and one Follower in `us-west-1`.
    • `FOLLOWER_CONSTRAINTS='{"+region=us-east-1,+type=scale-node": 1,"+region=us-west-1": 1}';` means placing one Follower on a node that is located in the `us-east-1` region and has the `type` label as `scale-node`, and one Follower in `us-west-1`.
    The dictionary format supports each key starting with `+` or `-` and allows you to configure the special `#reject-leader` attribute. For example, `FOLLOWER_CONSTRAINTS='{"+region=us-east-1":1, "+region=us-east-2": 2, "+region=us-west-1,#reject-leader": 1}'` means that the Leaders elected in `us-west-1` will be evicted as much as possible during disaster recovery.| +| List format | If a constraint to be specified applies to all replicas, you can use a key-value list format. Each key starts with `+` or `-`. For example:
    • `[+region=us-east-1]` means placing data on nodes that have a `region` label as `us-east-1`.
    • `[+region=us-east-1,-type=fault]` means placing data on nodes that have a `region` label as `us-east-1` but do not have a `type` label as `fault`.

    | +| Dictionary format | If you need to specify different numbers of replicas for different constraints, you can use the dictionary format. For example:
    • `FOLLOWER_CONSTRAINTS="{+region=us-east-1: 1,+region=us-east-2: 1,+region=us-west-1: 1}";` means placing one Follower in `us-east-1`, one Follower in `us-east-2`, and one Follower in `us-west-1`.
    • `FOLLOWER_CONSTRAINTS='{"+region=us-east-1,+type=scale-node": 1,"+region=us-west-1": 1}';` means placing one Follower on a node that is located in the `us-east-1` region and has the `type` label as `scale-node`, and one Follower in `us-west-1`.
    The dictionary format supports each key starting with `+` or `-` and allows you to configure the special `#evict-leader` attribute. For example, `FOLLOWER_CONSTRAINTS='{"+region=us-east-1":1, "+region=us-east-2": 2, "+region=us-west-1,#evict-leader": 1}'` means that the Leaders elected in `us-west-1` will be evicted as much as possible during disaster recovery.| > **Note:** > @@ -413,10 +413,10 @@ CREATE PLACEMENT POLICY deploy221_primary_east1 LEADER_CONSTRAINTS="[+region=us- After this placement policy is created and attached to the desired data, the Raft Leader replicas of the data will be placed in the `us-east-1` region specified by the `LEADER_CONSTRAINTS` option, while other replicas of the data will be placed in regions specified by the `FOLLOWER_CONSTRAINTS` option. Note that if the cluster fails, such as a node outage in the `us-east-1` region, a new Leader will still be elected from other regions, even if these regions are specified in `FOLLOWER_CONSTRAINTS`. In other words, ensuring service availability takes the highest priority. -In the event of a failure in the `us-east-1` region, if you do not want to place new Leaders in `us-west-1`, you can configure a special `reject-leader` attribute to evict the newly elected Leaders in that region: +In the event of a failure in the `us-east-1` region, if you do not want to place new Leaders in `us-west-1`, you can configure a special `evict-leader` attribute to evict the newly elected Leaders in that region: ```sql -CREATE PLACEMENT POLICY deploy221_primary_east1 LEADER_CONSTRAINTS="[+region=us-east-1]" FOLLOWER_CONSTRAINTS='{"+region=us-east-1": 1, "+region=us-east-2": 2, "+region=us-west-1,#reject-leader": 1}'; +CREATE PLACEMENT POLICY deploy221_primary_east1 LEADER_CONSTRAINTS="[+region=us-east-1]" FOLLOWER_CONSTRAINTS='{"+region=us-east-1": 1, "+region=us-east-2": 2, "+region=us-west-1,#evict-leader": 1}'; ``` #### Use `PRIMARY_REGION` diff --git a/post-installation-check.md b/post-installation-check.md index 020cb67954155..606bf40322984 100644 --- a/post-installation-check.md +++ b/post-installation-check.md @@ -1,7 +1,6 @@ --- title: Check Cluster Status summary: Learn how to check the running status of the TiDB cluster. -aliases: ['/docs/dev/post-installation-check/'] --- # Check Cluster Status @@ -63,7 +62,7 @@ The following information indicates successful login: ```sql Welcome to the MySQL monitor. Commands end with ; or \g. Your MySQL connection id is 3 -Server version: 8.0.11-TiDB-v7.4.0 TiDB Server (Apache License 2.0) Community Edition, MySQL 8.0 compatible +Server version: 8.0.11-TiDB-v{{{ .tidb-version }}} TiDB Server (Apache License 2.0) Community Edition, MySQL 8.0 compatible Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. Oracle is a registered trademark of Oracle Corporation and/or its affiliates. Other names may be trademarks of their respective diff --git a/predicate-push-down.md b/predicate-push-down.md index dbe6622d90101..de18a6981c069 100644 --- a/predicate-push-down.md +++ b/predicate-push-down.md @@ -1,7 +1,6 @@ --- title: Predicates Push Down summary: Introduce one of the TiDB's logic optimization rules—Predicate Push Down (PPD). -aliases: ['/tidb/dev/predicates-push-down'] --- # Predicates Push Down (PPD) diff --git a/privilege-management.md b/privilege-management.md index 39bf87ad5dc7b..71e3cac5dcec0 100644 --- a/privilege-management.md +++ b/privilege-management.md @@ -1,7 +1,6 @@ --- title: Privilege Management summary: Learn how to manage the privilege. -aliases: ['/docs/dev/privilege-management/','/docs/dev/reference/security/privilege-system/'] --- # Privilege Management @@ -299,7 +298,7 @@ Requires the `CREATE VIEW` privilege. ### DROP DATABASE -Requires the `DROP` privilege for the table. +Requires the `DROP` privilege for the database. ### DROP INDEX @@ -345,7 +344,7 @@ Requires the `INSERT` and `SELECT` privileges for the table. `SHOW GRANTS` requires the `SELECT` privilege to the `mysql` database. If the target user is current user, `SHOW GRANTS` does not require any privilege. -`SHOW PROCESSLIST` requires the `SUPER` privilege to show connections belonging to other users. +`SHOW PROCESSLIST` requires the `PROCESS` privilege to show connections belonging to other users. `SHOW IMPORT JOB` requires the `SUPER` privilege to show connections belonging to other users. Otherwise, it only shows jobs created by the current user. diff --git a/production-deployment-using-tiup.md b/production-deployment-using-tiup.md index d9b8c93203c01..b28ed819f0d79 100644 --- a/production-deployment-using-tiup.md +++ b/production-deployment-using-tiup.md @@ -1,7 +1,6 @@ --- title: Deploy a TiDB Cluster Using TiUP summary: Learn how to easily deploy a TiDB cluster using TiUP. -aliases: ['/docs/dev/production-deployment-using-tiup/','/docs/dev/how-to/deploy/orchestrated/tiup/','/docs/dev/tiflash/deploy-tiflash/','/docs/dev/reference/tiflash/deploy/','/tidb/dev/deploy-tidb-from-dbdeployer/','/docs/dev/deploy-tidb-from-dbdeployer/','/docs/dev/how-to/get-started/deploy-tidb-from-dbdeployer/','/tidb/dev/deploy-tidb-from-homebrew/','/docs/dev/deploy-tidb-from-homebrew/','/docs/dev/how-to/get-started/deploy-tidb-from-homebrew/','/tidb/dev/production-offline-deployment-using-tiup','/docs/dev/production-offline-deployment-using-tiup/','/tidb/dev/deploy-tidb-from-binary','/tidb/dev/production-deployment-from-binary-tarball','/tidb/dev/test-deployment-from-binary-tarball','/tidb/dev/deploy-test-cluster-using-docker-compose','/tidb/dev/test-deployment-using-docker'] --- # Deploy a TiDB Cluster Using TiUP @@ -23,6 +22,10 @@ You can deploy TiUP on the control machine in either of the two ways: online dep ### Deploy TiUP online +> **Note:** +> +> If the TiUP environment switches to offline, refer to [Deploy TiUP offline](#deploy-tiup-offline) for deployment. Otherwise, TiUP cannot work properly. + Log in to the control machine using a regular user account (take the `tidb` user as an example). Subsequent TiUP installation and cluster management can be performed by the `tidb` user. 1. Install TiUP by running the following command: @@ -83,9 +86,21 @@ Perform the following steps in this section to deploy a TiDB cluster offline usi #### Prepare the TiUP offline component package -Method 1: On the [official download page](https://www.pingcap.com/download/), select the offline mirror package (TiUP offline package included) of the target TiDB version. Note that you need to download the server package and toolkit package at the same time. +**Method 1**: Download the offline binary packages (TiUP offline package included) of the target TiDB version using the following links. You need to download both the server and toolkit packages. Note that your downloading means you agree to the [Privacy Policy](https://www.pingcap.com/privacy-policy/). + +``` +https://download.pingcap.com/tidb-community-server-{version}-linux-{arch}.tar.gz +``` + +``` +https://download.pingcap.com/tidb-community-toolkit-{version}-linux-{arch}.tar.gz +``` + +> **Tip:** +> +> `{version}` in the link indicates the version number of TiDB and `{arch}` indicates the architecture of the system, which can be `amd64` or `arm64`. For example, the download link for `v7.5.0` in the `amd64` architecture is `https://download.pingcap.com/tidb-community-toolkit-v7.5.0-linux-amd64.tar.gz`. -Method 2: Manually pack an offline component package using `tiup mirror clone`. The detailed steps are as follows: +**Method 2**: Manually pack an offline component package using `tiup mirror clone`. The detailed steps are as follows: 1. Install the TiUP package manager online. @@ -197,7 +212,7 @@ The `local_install.sh` script automatically runs the `tiup mirror set tidb-commu #### Merge offline packages -If you download the offline packages from the [official download page](https://www.pingcap.com/download/), you need to merge the server package and the toolkit package into an offline mirror. If you manually package the offline component packages using the `tiup mirror clone` command, you can skip this step. +If you download the offline packages via download links, you need to merge the server package and the toolkit package into an offline mirror. If you manually package the offline component packages using the `tiup mirror clone` command, you can skip this step. Run the following commands to merge the offline toolkit package into the server package directory: @@ -291,10 +306,10 @@ The following examples cover seven common scenarios. You need to modify the conf For more configuration description, see the following configuration examples: -- [TiDB `config.toml.example`](https://github.com/pingcap/tidb/blob/master/pkg/config/config.toml.example) -- [TiKV `config.toml.example`](https://github.com/tikv/tikv/blob/master/etc/config-template.toml) -- [PD `config.toml.example`](https://github.com/pingcap/pd/blob/master/conf/config.toml) -- [TiFlash `config.toml.example`](https://github.com/pingcap/tiflash/blob/master/etc/config-template.toml) +- [TiDB `config.toml.example`](https://github.com/pingcap/tidb/blob/release-7.5/pkg/config/config.toml.example) +- [TiKV `config.toml.example`](https://github.com/tikv/tikv/blob/release-7.5/etc/config-template.toml) +- [PD `config.toml.example`](https://github.com/pingcap/pd/blob/release-7.5/conf/config.toml) +- [TiFlash `config.toml.example`](https://github.com/pingcap/tiflash/blob/release-7.5/etc/config-template.toml) ## Step 4. Run the deployment command @@ -334,13 +349,13 @@ Before you run the `deploy` command, use the `check` and `check --apply` command {{< copyable "shell-regular" >}} ```shell - tiup cluster deploy tidb-test v7.4.0 ./topology.yaml --user root [-p] [-i /home/root/.ssh/gcp_rsa] + tiup cluster deploy tidb-test v{{{ .tidb-version }}} ./topology.yaml --user root [-p] [-i /home/root/.ssh/gcp_rsa] ``` In the `tiup cluster deploy` command above: - `tidb-test` is the name of the TiDB cluster to be deployed. -- `v7.4.0` is the version of the TiDB cluster to be deployed. You can see the latest supported versions by running `tiup list tidb`. +- `v{{{ .tidb-version }}}` is the version of the TiDB cluster to be deployed. You can see the latest supported versions by running `tiup list tidb`. - `topology.yaml` is the initialization configuration file. - `--user root` indicates logging into the target machine as the `root` user to complete the cluster deployment. The `root` user is expected to have `ssh` and `sudo` privileges to the target machine. Alternatively, you can use other users with `ssh` and `sudo` privileges to complete the deployment. - `[-i]` and `[-p]` are optional. If you have configured login to the target machine without password, these parameters are not required. If not, choose one of the two parameters. `[-i]` is the private key of the root user (or other users specified by `--user`) that has access to the target machine. `[-p]` is used to input the user password interactively. diff --git a/quick-start-with-tidb.md b/quick-start-with-tidb.md index 28843532ddb81..c2596ba1aa8d1 100644 --- a/quick-start-with-tidb.md +++ b/quick-start-with-tidb.md @@ -1,7 +1,6 @@ --- title: Quick Start Guide for the TiDB Database Platform summary: Learn how to quickly get started with the TiDB platform and see if TiDB is the right choice for you. -aliases: ['/docs/dev/quick-start-with-tidb/','/docs/dev/test-deployment-using-docker/'] --- # Quick Start Guide for the TiDB Database Platform @@ -15,7 +14,7 @@ In addition, you can try out TiDB features on [TiDB Playground](https://play.tid > **Note:** > -> The deployment method provided in this guide is **ONLY FOR** quick start, **NOT FOR** production. +> The deployment method provided in this guide is **ONLY FOR** quick start, **NOT FOR** production or comprehensive functionality and stability testing. > > - To deploy a self-hosted production cluster, see the [production installation guide](/production-deployment-using-tiup.md). > - To deploy TiDB on Kubernetes, see [Get Started with TiDB on Kubernetes](https://docs.pingcap.com/tidb-in-kubernetes/stable/get-started). @@ -81,10 +80,10 @@ As a distributed system, a basic TiDB test cluster usually consists of 2 TiDB in {{< copyable "shell-regular" >}} ```shell - tiup playground v7.4.0 --db 2 --pd 3 --kv 3 + tiup playground v{{{ .tidb-version }}} --db 2 --pd 3 --kv 3 ``` - The command downloads a version cluster to the local machine and starts it, such as v7.4.0. To view the latest version, run `tiup list tidb`. + The command downloads a version cluster to the local machine and starts it, such as v{{{ .tidb-version }}}. To view the latest version, run `tiup list tidb`. This command returns the access methods of the cluster: @@ -202,10 +201,10 @@ As a distributed system, a basic TiDB test cluster usually consists of 2 TiDB in {{< copyable "shell-regular" >}} ```shell - tiup playground v7.4.0 --db 2 --pd 3 --kv 3 + tiup playground v{{{ .tidb-version }}} --db 2 --pd 3 --kv 3 ``` - The command downloads a version cluster to the local machine and starts it, such as v7.4.0. To view the latest version, run `tiup list tidb`. + The command downloads a version cluster to the local machine and starts it, such as v{{{ .tidb-version }}}. To view the latest version, run `tiup list tidb`. This command returns the access methods of the cluster: @@ -437,7 +436,7 @@ Other requirements for the target machine include: ``` - ``: Set the cluster name - - ``: Set the TiDB cluster version, such as `v7.4.0`. You can see all the supported TiDB versions by running the `tiup list tidb` command + - ``: Set the TiDB cluster version, such as `v{{{ .tidb-version }}}`. You can see all the supported TiDB versions by running the `tiup list tidb` command - `-p`: Specify the password used to connect to the target machine. > **Note:** diff --git a/read-historical-data.md b/read-historical-data.md index bb532cb7db618..0a75f4fd76c6e 100644 --- a/read-historical-data.md +++ b/read-historical-data.md @@ -1,7 +1,6 @@ --- title: Read Historical Data Using the System Variable `tidb_snapshot` summary: Learn about how TiDB reads data from history versions using the system variable `tidb_snapshot`. -aliases: ['/docs/dev/read-historical-data/','/docs/dev/how-to/get-started/read-historical-data/'] --- # Read Historical Data Using the System Variable `tidb_snapshot` diff --git a/releases/release-1.0-ga.md b/releases/release-1.0-ga.md index dd572e546f6f5..b97ea2bc7d6e2 100644 --- a/releases/release-1.0-ga.md +++ b/releases/release-1.0-ga.md @@ -1,6 +1,6 @@ --- title: TiDB 1.0 release notes -aliases: ['/docs/dev/releases/release-1.0-ga/','/docs/dev/releases/ga/'] +summary: TiDB 1.0 is released with a focus on MySQL compatibility, SQL optimization, stability, and performance. It includes enhancements to the SQL query optimizer, internal data format optimization, and support for various operators. PD now supports read flow based balancing and setting store weight. TiKV has improved coprocessor support and performance, and added a Debug API. Special thanks to enterprises, open source software, and individual contributors for their support. --- # TiDB 1.0 Release Notes diff --git a/releases/release-1.0.1.md b/releases/release-1.0.1.md index eec2891e4bba0..615989a00d889 100644 --- a/releases/release-1.0.1.md +++ b/releases/release-1.0.1.md @@ -1,6 +1,6 @@ --- title: TiDB 1.0.1 Release Notes -aliases: ['/docs/dev/releases/release-1.0.1/','/docs/dev/releases/101/'] +summary: TiDB 1.0.1 was released on November 1, 2017. Updates include support for canceling DDL Job, optimizing the `IN` expression, correcting the result type of the `Show` statement, supporting log slow query into a separate log file, and fixing bugs. TiKV now supports flow control with write bytes, reduces Raft allocation, increases coprocessor stack size to 10MB, and removes the useless log from the coprocessor. --- # TiDB 1.0.1 Release Notes diff --git a/releases/release-1.0.2.md b/releases/release-1.0.2.md index 19db611b557d3..20705c1d1760c 100644 --- a/releases/release-1.0.2.md +++ b/releases/release-1.0.2.md @@ -1,6 +1,6 @@ --- title: TiDB 1.0.2 Release Notes -aliases: ['/docs/dev/releases/release-1.0.2/','/docs/dev/releases/102/'] +summary: TiDB 1.0.2 was released on November 13, 2017. Updates include optimized cost estimation for index point query, support for Alter Table Add Column syntax, and improved query optimization. Placement Driver (PD) scheduling stability was enhanced, and TiKV now supports table splitting and limits key length to 4 KB. Other improvements include more accurate read traffic statistics and bug fixes for LIKE behavior and do_div_mod bug. --- # TiDB 1.0.2 Release Notes diff --git a/releases/release-1.0.3.md b/releases/release-1.0.3.md index ac57f7eea1f47..064e38ba5b4f1 100644 --- a/releases/release-1.0.3.md +++ b/releases/release-1.0.3.md @@ -1,6 +1,6 @@ --- title: TiDB 1.0.3 Release Notes -aliases: ['/docs/dev/releases/release-1.0.3/','/docs/dev/releases/103/'] +summary: TiDB 1.0.3 was released on November 28, 2017. Updates include performance optimization, new configuration options, and bug fixes. PD now supports adding more schedulers using API, and TiKV has fixed deadlock and leader value issues. To upgrade from 1.0.2 to 1.0.3, follow the rolling upgrade order of PD, TiKV, and TiDB. --- # TiDB 1.0.3 Release Notes diff --git a/releases/release-1.0.4.md b/releases/release-1.0.4.md index 48b6acbfac36d..522c2039c96b3 100644 --- a/releases/release-1.0.4.md +++ b/releases/release-1.0.4.md @@ -1,6 +1,6 @@ --- title: TiDB 1.0.4 Release Notes -aliases: ['/docs/dev/releases/release-1.0.4/','/docs/dev/releases/104/'] +summary: TiDB 1.0.4 was released on December 11, 2017. Updates include speed improvements, performance enhancements, and fixes for potential issues in TiDB and TiKV. To upgrade from 1.0.3 to 1.0.4, follow the rolling upgrade order of PD, TiKV, and TiDB. --- # TiDB 1.0.4 Release Notes diff --git a/releases/release-1.0.5.md b/releases/release-1.0.5.md index 6a4e6718eb535..5affec8ed2a50 100644 --- a/releases/release-1.0.5.md +++ b/releases/release-1.0.5.md @@ -1,6 +1,6 @@ --- title: TiDB 1.0.5 Release Notes -aliases: ['/docs/dev/releases/release-1.0.5/','/docs/dev/releases/105/'] +summary: TiDB 1.0.5 was released on December 26, 2017. Updates include adding max value for Auto_Increment ID, fixing goroutine leak, supporting output of slow queries to separate file, loading TimeZone variable from TiKV, and more. PD fixes include balancing leaders and potential panic during bootstrapping. TiKV fixes slow CPU ID retrieval and supports dynamic-level-bytes parameter. Upgrade order is PD -> TiKV -> TiDB. --- # TiDB 1.0.5 Release Notes diff --git a/releases/release-1.0.6.md b/releases/release-1.0.6.md index b744c148887d7..a794b639cbf31 100644 --- a/releases/release-1.0.6.md +++ b/releases/release-1.0.6.md @@ -1,6 +1,6 @@ --- title: TiDB 1.0.6 Release Notes -aliases: ['/docs/dev/releases/release-1.0.6/','/docs/dev/releases/106/'] +summary: TiDB 1.0.6 was released on January 08, 2018. Updates include support for Alter Table Auto_Increment syntax, fixing bugs in Cost Based computation and Null Json issue, and support for extension syntax to shard implicit row ID. Other updates include fixing potential DDL issue, considering timezone setting in certain functions, and support for SEPARATOR syntax in GROUP_CONCAT function. PD fixed store selection problem of hot-region scheduler. To upgrade from 1.0.5 to 1.0.6, follow the rolling upgrade order of PD, TiKV, TiDB. --- # TiDB 1.0.6 Release Notes diff --git a/releases/release-1.0.7.md b/releases/release-1.0.7.md index c8b9362d3fb67..84e15126d0eb9 100644 --- a/releases/release-1.0.7.md +++ b/releases/release-1.0.7.md @@ -1,6 +1,6 @@ --- title: TiDB 1.0.7 Release Notes -aliases: ['/docs/dev/releases/release-1.0.7/','/docs/dev/releases/107/'] +summary: TiDB 1.0.7 is released with various updates including optimization of commands, fixing data race and resource leak issues, adding session variable for log query control, and improving stability of test results. PD and TiKV also have updates to fix scheduling loss issues, compatibility issues, and add support for table scan and remote mode in tikv-ctl. To upgrade from 1.0.6 to 1.0.7, follow the rolling upgrade order of PD, TiKV, and TiDB. --- # TiDB 1.0.7 Release Notes diff --git a/releases/release-1.0.8.md b/releases/release-1.0.8.md index 2f8e1539c98d7..c4bbfed655891 100644 --- a/releases/release-1.0.8.md +++ b/releases/release-1.0.8.md @@ -1,6 +1,6 @@ --- title: TiDB 1.0.8 Release Notes -aliases: ['/docs/dev/releases/release-1.0.8/','/docs/dev/releases/108/'] +summary: TiDB 1.0.8 is released with updates including fixes for various issues, performance optimizations, and stability improvements. PD and TiKV also have updates related to reducing lock overheat, fixing leader selection issues, and improving starting speed. To upgrade, follow the order of PD -> TiKV -> TiDB. --- # TiDB 1.0.8 Release Notes diff --git a/releases/release-1.1-alpha.md b/releases/release-1.1-alpha.md index a9cb91163f7c4..1eed9eefc6123 100644 --- a/releases/release-1.1-alpha.md +++ b/releases/release-1.1-alpha.md @@ -1,6 +1,6 @@ --- title: TiDB 1.1 Alpha Release Notes -aliases: ['/docs/dev/releases/release-1.1-alpha/','/docs/dev/releases/11alpha/'] +summary: TiDB 1.1 Alpha, released on January 19, 2018, brings significant improvements in MySQL compatibility, SQL optimization, stability, and performance. Key updates include enhanced SQL parser, query optimizer, and executor, as well as server support for the PROXY protocol. PD now offers more APIs, TLS support, and improved scheduling, while TiKV introduces Raft learner support, TLS, and performance optimizations. Additionally, it enhances data recovery tools and improves flow control mechanisms. --- # TiDB 1.1 Alpha Release Notes diff --git a/releases/release-1.1-beta.md b/releases/release-1.1-beta.md index fece9cc567d44..7574880b58532 100644 --- a/releases/release-1.1-beta.md +++ b/releases/release-1.1-beta.md @@ -1,6 +1,6 @@ --- title: TiDB 1.1 Beta Release Notes -aliases: ['/docs/dev/releases/release-1.1-beta/','/docs/dev/releases/11beta/'] +summary: TiDB 1.1 Beta, released on February 24, 2018, brings significant improvements in MySQL compatibility, SQL optimization, stability, and performance. Key updates include more monitoring metrics, enhanced MySQL syntax compatibility, improved query optimization, and stability fixes. PD introduces new debug interfaces, priority settings, and performance optimizations. TiKV adds support for resolving locks in batches, GC concurrency, and more recovery operations in `tikv-ctl`. Overall, TiDB 1.1 Beta shows great improvement in test results and stability. --- # TiDB 1.1 Beta Release Notes diff --git a/releases/release-2.0-ga.md b/releases/release-2.0-ga.md index b35e6a39f0598..fd482ba357f16 100644 --- a/releases/release-2.0-ga.md +++ b/releases/release-2.0-ga.md @@ -1,6 +1,6 @@ --- title: TiDB 2.0 Release Notes -aliases: ['/docs/dev/releases/release-2.0-ga/','/docs/dev/releases/2.0ga/'] +summary: TiDB 2.0 GA release on April 27, 2018, brings improvements in MySQL compatibility, SQL optimizer, executor, and stability. Key updates include compact data structure for memory usage reduction, Stream Aggregation operator for empty GROUP BY clause, and support for more MySQL syntaxes. TiKV features include `Region Merge`, `Raw DeleteRange` API, and improved read performance using `ReadPool`. TiSpark 1.0 GA provides distributed computing of TiDB data using Apache Spark, with support for gRPC communication framework, calculation pushdown, index related support, cost-based optimization, and multiple Spark interfaces. --- # TiDB 2.0 Release Notes @@ -67,7 +67,7 @@ On April 27, 2018, TiDB 2.0 GA is released! Compared with TiDB 1.0, this release - Optimize the scheduling policies to prevent the disks from becoming full when the space of TiKV nodes is insufficient - Improve the scheduling efficiency of the balance-leader scheduler - Reduce the scheduling overhead of the balance-region scheduler - - Optimize the execution efficiency of the the hot-region scheduler + - Optimize the execution efficiency of the hot-region scheduler - Operations interface and configuration - Support TLS - Support prioritizing the PD leaders diff --git a/releases/release-2.0-rc.1.md b/releases/release-2.0-rc.1.md index aaff6e7e006df..3e751aff0c482 100644 --- a/releases/release-2.0-rc.1.md +++ b/releases/release-2.0-rc.1.md @@ -1,6 +1,6 @@ --- title: TiDB 2.0 RC1 Release Notes -aliases: ['/docs/dev/releases/release-2.0-rc.1/','/docs/dev/releases/2rc1/'] +summary: TiDB 2.0 RC1, released on March 9, 2018, brings improvements in MySQL compatibility, SQL optimization, and stability. Key updates include memory usage limitation for SQL statements, Stream Aggregate operator support, configuration file validation, and HTTP API for configuration information. TiDB also enhances MySQL syntax compatibility, optimizer, and Boolean field length. PD sees logic and performance optimizations, while TiKV fixes gRPC call and adds gRPC APIs for metrics. Additionally, TiKV checks SSD usage, optimizes read performance, and improves metrics usage. --- # TiDB 2.0 RC1 Release Notes diff --git a/releases/release-2.0-rc.3.md b/releases/release-2.0-rc.3.md index 505fe641d6276..790bdd6c134eb 100644 --- a/releases/release-2.0-rc.3.md +++ b/releases/release-2.0-rc.3.md @@ -1,6 +1,6 @@ --- title: TiDB 2.0 RC3 Release Notes -aliases: ['/docs/dev/releases/release-2.0-rc.3/','/docs/dev/releases/2rc3/'] +summary: TiDB 2.0 RC3, released on March 23, 2018, brings improvements in MySQL compatibility, SQL optimization, and stability. It includes fixes for various issues, optimizations for execution speed, memory control, and DDL job management. PD now supports Region Merge and has optimizations for leader balance and abnormal Regions. TiKV also supports Region Merge, Raft snapshot process, and streaming in Coprocessor, with various improvements in space management and data recovery. --- # TiDB 2.0 RC3 Release Notes diff --git a/releases/release-2.0-rc.4.md b/releases/release-2.0-rc.4.md index cd6b668554c06..74ef45ec37f91 100644 --- a/releases/release-2.0-rc.4.md +++ b/releases/release-2.0-rc.4.md @@ -1,6 +1,6 @@ --- title: TiDB 2.0 RC4 Release Notes -aliases: ['/docs/dev/releases/release-2.0-rc.4/','/docs/dev/releases/2rc4/'] +summary: TiDB 2.0 RC4, released on March 30, 2018, brings improvements in MySQL compatibility, SQL optimization, and stability. Key updates include support for various syntax, bug fixes, and performance optimizations in TiDB, PD, and TiKV. Notable changes include manual Region splitting in PD, memory usage limitation in TiKV, and support for data pattern import. Overall, the release focuses on enhancing functionality and addressing performance issues. --- # TiDB 2.0 RC4 Release Notes diff --git a/releases/release-2.0-rc.5.md b/releases/release-2.0-rc.5.md index 34ba7824c992b..8e5d249bf48e3 100644 --- a/releases/release-2.0-rc.5.md +++ b/releases/release-2.0-rc.5.md @@ -1,6 +1,6 @@ --- title: TiDB 2.0 RC5 Release Notes -aliases: ['/docs/dev/releases/release-2.0-rc.5/','/docs/dev/releases/2rc5/'] +summary: TiDB 2.0 RC5 was released on April 17, 2018, with improvements in MySQL compatibility, SQL optimization, and stability. Fixes and optimizations were made to TiDB, PD, and TiKV components, including support for Raft Learner, reducing scheduling overhead, and adding new batch operations. The release also addressed issues related to memory usage, error reporting, and configuration adjustments. --- # TiDB 2.0 RC5 Release Notes diff --git a/releases/release-2.0.1.md b/releases/release-2.0.1.md index 4270f3609892e..2bd3d0a4294cd 100644 --- a/releases/release-2.0.1.md +++ b/releases/release-2.0.1.md @@ -1,6 +1,6 @@ --- title: TiDB 2.0.1 Release Notes -aliases: ['/docs/dev/releases/release-2.0.1/','/docs/dev/releases/201/'] +summary: TiDB 2.0.1 was released on May 16, 2018, with improvements in MySQL compatibility and system stability. Updates include real-time progress for 'Add Index', a new session variable for automatic statistics update, bug fixes, compatibility improvements, and behavior changes. PD added a new scheduler, optimized region balancing, and fixed various issues. TiKV fixed issues related to reading, thread calls, raftstore blocking, and split causing dirty read. Overall, the release focuses on enhancing performance, stability, and compatibility. --- # TiDB 2.0.1 Release Notes diff --git a/releases/release-2.0.10.md b/releases/release-2.0.10.md index 7e0cfe9349e21..a6f02f70a8e92 100644 --- a/releases/release-2.0.10.md +++ b/releases/release-2.0.10.md @@ -1,6 +1,6 @@ --- title: TiDB 2.0.10 Release Notes -aliases: ['/docs/dev/releases/release-2.0.10/','/docs/dev/releases/2.0.10/'] +summary: TiDB 2.0.10 and TiDB Ansible 2.0.10 were released on December 18, 2018. The release includes improvements in system compatibility and stability. Fixes include issues with DDL jobs, ORDER BY and UNION clauses, UNCOMPRESS function, ANSI_QUOTES SQL_MODE, select results, and more. PD fixes a possible RaftCluster deadlock issue, while TiKV optimizes leader transfer and fixes redundant Region heartbeats. --- # TiDB 2.0.10 Release Notes diff --git a/releases/release-2.0.11.md b/releases/release-2.0.11.md index 63f60a9a15cb0..2e49ec415e332 100644 --- a/releases/release-2.0.11.md +++ b/releases/release-2.0.11.md @@ -1,6 +1,6 @@ --- title: TiDB 2.0.11 Release Notes -aliases: ['/docs/dev/releases/release-2.0.11/','/docs/dev/releases/2.0.11/'] +summary: TiDB 2.0.11 and TiDB Ansible 2.0.11 were released on January 3, 2019. The release includes improvements in system compatibility and stability. Fixes include handling errors when PD is in an abnormal condition, compatibility issues with MySQL, error message reporting, prefix index range, and panic issues with the `UPDATE` statement. TiKV also fixed two issues related to Region merge. --- # TiDB 2.0.11 Release Notes diff --git a/releases/release-2.0.2.md b/releases/release-2.0.2.md index ed9665c7ec6da..63e99a3ac4be1 100644 --- a/releases/release-2.0.2.md +++ b/releases/release-2.0.2.md @@ -1,6 +1,6 @@ --- title: TiDB 2.0.2 Release Notes -aliases: ['/docs/dev/releases/release-2.0.2/','/docs/dev/releases/202/'] +summary: TiDB 2.0.2 was released on May 21, 2018, with improvements in system stability. The release includes fixes for Decimal division expression, support for `USE INDEX` syntax in `Delete` statement, and timeout mechanism for writing Binlog in TiDB. PD now filters disconnected nodes in balance leader scheduler, modifies transfer leader operator timeout, and fixes scheduling issues. TiKV fixes Raft log printing, supports configuring gRPC parameters, leader election timeout range, and resolves snapshot intermediate file deletion issue. --- # TiDB 2.0.2 Release Notes diff --git a/releases/release-2.0.3.md b/releases/release-2.0.3.md index 5121baea898ac..dd8c8216d02d3 100644 --- a/releases/release-2.0.3.md +++ b/releases/release-2.0.3.md @@ -1,6 +1,6 @@ --- title: TiDB 2.0.3 Release Notes -aliases: ['/docs/dev/releases/release-2.0.3/','/docs/dev/releases/203/'] +summary: TiDB 2.0.3 was released on June 1, 2018, with improvements in system compatibility and stability. It includes various fixes and optimizations for TiDB, PD, and TiKV. Some highlights are support for modifying log level online, fixing issues with unique index and `ON DUPLICATE KEY UPDATE`, and addressing panic issues in specific conditions. --- # TiDB 2.0.3 Release Notes diff --git a/releases/release-2.0.4.md b/releases/release-2.0.4.md index 95a024d2c3010..705a6ca847240 100644 --- a/releases/release-2.0.4.md +++ b/releases/release-2.0.4.md @@ -1,6 +1,6 @@ --- title: TiDB 2.0.4 Release Notes -aliases: ['/docs/dev/releases/release-2.0.4/','/docs/dev/releases/204/'] +summary: TiDB 2.0.4 was released on June 15, 2018, with improvements in system compatibility and stability. It includes various enhancements and fixes for TiDB, PD, and TiKV. Some highlights for TiDB are support for `ALTER TABLE t DROP COLUMN a CASCADE` syntax, refining statement type display, and fixing issues related to data conversion and result order. PD now has improved behavior for the `max-pending-peer-count` argument, while TiKV includes the addition of the RocksDB `PerfContext` interface and fixes for slow `reverse-seek` and crash issues. --- # TiDB 2.0.4 Release Notes diff --git a/releases/release-2.0.5.md b/releases/release-2.0.5.md index dc8f7689d130f..2ebfa0667d0f1 100644 --- a/releases/release-2.0.5.md +++ b/releases/release-2.0.5.md @@ -1,6 +1,6 @@ --- title: TiDB 2.0.5 Release Notes -aliases: ['/docs/dev/releases/release-2.0.5/','/docs/dev/releases/205/'] +summary: TiDB 2.0.5 was released on July 6, 2018, with improvements in system compatibility and stability. New features include the `tidb_disable_txn_auto_retry` system variable. Bug fixes address issues with user login, data insertion, and command compatibility. PD and TiKV also received fixes for various issues. --- # TiDB 2.0.5 Release Notes diff --git a/releases/release-2.0.6.md b/releases/release-2.0.6.md index 1c389a9079f5f..d51b6be7fe737 100644 --- a/releases/release-2.0.6.md +++ b/releases/release-2.0.6.md @@ -1,6 +1,6 @@ --- title: TiDB 2.0.6 Release Notes -aliases: ['/docs/dev/releases/release-2.0.6/','/docs/dev/releases/206/'] +summary: TiDB 2.0.6 was released on August 6, 2018, with improvements in system compatibility and stability. The release includes various improvements and bug fixes for TiDB and TiKV. Some notable improvements include reducing transaction conflicts, improving row count estimation accuracy, and adding a recover mechanism for panics during the execution of `ANALYZE TABLE`. Bug fixes address issues such as incompatible `DROP USER` statement behavior, OOM errors for `INSERT`/`LOAD DATA` statements, and incorrect results for prefix index and `DECIMAL` operations. TiKV also sees improvements in scheduler slots, rollback transaction records, and RocksDB log file management, along with a fix for a crash issue during data type conversion. --- # TiDB 2.0.6 Release Notes diff --git a/releases/release-2.0.7.md b/releases/release-2.0.7.md index 9d3423c8e1715..b2491ad4be665 100644 --- a/releases/release-2.0.7.md +++ b/releases/release-2.0.7.md @@ -1,6 +1,6 @@ --- title: TiDB 2.0.7 Release Notes -aliases: ['/docs/dev/releases/release-2.0.7/','/docs/dev/releases/207/'] +summary: TiDB 2.0.7 was released on September 7, 2018, with improvements in system compatibility and stability. New features include the addition of the `PROCESSLIST` table in `information_schema`. Bug fixes address issues with index usage, join output, and query conditions. TiKV now opens the `dynamic-level-bytes` parameter by default to reduce space amplification, and updates approximate size and keys count after region merging. --- # TiDB 2.0.7 Release Notes diff --git a/releases/release-2.0.8.md b/releases/release-2.0.8.md index 7d5d6c5c6c7a2..61d7e5b014543 100644 --- a/releases/release-2.0.8.md +++ b/releases/release-2.0.8.md @@ -1,6 +1,6 @@ --- title: TiDB 2.0.8 Release Notes -aliases: ['/docs/dev/releases/release-2.0.8/','/docs/dev/releases/208/'] +summary: TiDB 2.0.8 was released on October 16, 2018, with improvements in system compatibility and stability. The release includes various bug fixes for TiDB and TiKV, addressing issues related to AUTO-ID, etcd session recovery, time zone handling, memory leaks, and join conversions. The TiKV bug fix resolves the increasing memory consumption by Raftstore EntryCache when a node goes down. --- # TiDB 2.0.8 Release Notes diff --git a/releases/release-2.0.9.md b/releases/release-2.0.9.md index 4caed728aa23a..6c25dec981390 100644 --- a/releases/release-2.0.9.md +++ b/releases/release-2.0.9.md @@ -1,6 +1,6 @@ --- title: TiDB 2.0.9 Release Notes -aliases: ['/docs/dev/releases/release-2.0.9/','/docs/dev/releases/209/'] +summary: TiDB 2.0.9 was released on November 19, 2018, with significant improvements in system compatibility and stability. The release includes fixes for various issues, such as empty statistics histogram, panic issue with UNION ALL statement, stack overflow issue, and support for specifying utf8mb4 character set. PD and TiKV also received fixes for issues related to server startup failure and interface limits. --- # TiDB 2.0.9 Release Notes diff --git a/releases/release-2.1-beta.md b/releases/release-2.1-beta.md index afc880c3ac23a..03bc7c8756287 100644 --- a/releases/release-2.1-beta.md +++ b/releases/release-2.1-beta.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1 Beta Release Notes -aliases: ['/docs/dev/releases/release-2.1-beta/','/docs/dev/releases/21beta/'] +summary: TiDB 2.1 Beta release includes improvements in stability, SQL optimizer, statistics, and execution engine. It supports more MySQL syntax, decreases memory usage, and optimizes DDL and DML statements. PD enables Raft PreVote, optimizes scheduler issues, and adds metrics. TiKV upgrades Rust, adds metrics, and improves performance. Compatibility notes include not supporting rollback to v2.0.x and enabling raft learner by default in the new version. --- # TiDB 2.1 Beta Release Notes diff --git a/releases/release-2.1-ga.md b/releases/release-2.1-ga.md index 6b52b6ffd5718..dd49ff7ff40e0 100644 --- a/releases/release-2.1-ga.md +++ b/releases/release-2.1-ga.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1 GA Release Notes -aliases: ['/docs/dev/releases/release-2.1-ga/','/docs/dev/releases/2.1ga/'] +summary: TiDB 2.1 GA was released on November 30, 2018, with significant improvements in stability, performance, compatibility, and usability. The release includes optimizations in SQL optimizer, SQL executor, statistics, expressions, server, DDL, compatibility, Placement Driver (PD), TiKV, and tools. It also introduces TiDB Lightning for fast full data import and supports new TiDB Binlog. However, TiDB 2.1 does not support downgrading to v2.0.x or earlier due to the adoption of the new storage engine. Additionally, parallel DDL is enabled in TiDB 2.1, so clusters with TiDB version earlier than 2.0.1 cannot upgrade to 2.1 using rolling update. If upgrading from TiDB 2.0.6 or earlier to TiDB 2.1, ongoing DDL operations may slow down the upgrading process. --- # TiDB 2.1 GA Release Notes @@ -89,7 +89,7 @@ On November 30, 2018, TiDB 2.1 GA is released. See the following updates in this - Check the TiDB cluster information - - [Add the `auto_analyze_ratio` system variables to contorl the ratio of Analyze](/faq/sql-faq.md#whats-the-trigger-strategy-for-auto-analyze-in-tidb) + - [Add the `auto_analyze_ratio` system variables to control the ratio of Analyze](/faq/sql-faq.md#whats-the-trigger-strategy-for-auto-analyze-in-tidb) - [Add the `tidb_retry_limit` system variable to control the automatic retry times of transactions](/system-variables.md#tidb_retry_limit) diff --git a/releases/release-2.1-rc.1.md b/releases/release-2.1-rc.1.md index 441e93866302e..5b8db24d68d04 100644 --- a/releases/release-2.1-rc.1.md +++ b/releases/release-2.1-rc.1.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1 RC1 Release Notes -aliases: ['/docs/dev/releases/release-2.1-rc.1/','/docs/dev/releases/21rc1/'] +summary: TiDB 2.1 RC1 was released on August 24, 2018, with improvements in stability, SQL optimizer, statistics, and execution engine. The release includes fixes for various issues in SQL optimizer and execution engine. PD introduces version control, rolling update, and region merge features. TiKV supports batch split and row-based region splitting for improved efficiency. Overall, the release focuses on performance optimization and bug fixes. --- # TiDB 2.1 RC1 Release Notes diff --git a/releases/release-2.1-rc.2.md b/releases/release-2.1-rc.2.md index db9ec0f424bfc..82126e37091ba 100644 --- a/releases/release-2.1-rc.2.md +++ b/releases/release-2.1-rc.2.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1 RC2 Release Notes -aliases: ['/docs/dev/releases/release-2.1-rc.2/','/docs/dev/releases/21rc2/'] +summary: TiDB 2.1 RC2 was released on September 14, 2018, with improvements in stability, SQL optimizer, statistics, and execution engine. The release includes enhancements to SQL optimizer, SQL execution engine, statistics, server, compatibility, expressions, DML, DDL, TiKV Go Client, and Table Partition. PD features, improvements, and bug fixes are also included. TiKV performance, improvements, and bug fixes are part of the release as well. --- # TiDB 2.1 RC2 Release Notes diff --git a/releases/release-2.1-rc.3.md b/releases/release-2.1-rc.3.md index 9bed927dd202e..c6f8ca6e9bb5a 100644 --- a/releases/release-2.1-rc.3.md +++ b/releases/release-2.1-rc.3.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1 RC3 Release Notes -aliases: ['/docs/dev/releases/release-2.1-rc.3/','/docs/dev/releases/21rc3/'] +summary: TiDB 2.1 RC3 was released on September 29, 2018, with improvements in stability, compatibility, SQL optimizer, and execution engine. The release includes fixes and enhancements for SQL optimizer, execution engine, server, compatibility, expressions, DML, DDL, and PD. TiKV also received performance optimizations, new features, and bug fixes. --- # TiDB 2.1 RC3 Release Notes diff --git a/releases/release-2.1-rc.4.md b/releases/release-2.1-rc.4.md index 20844d8fea253..1f425fdfafcb1 100644 --- a/releases/release-2.1-rc.4.md +++ b/releases/release-2.1-rc.4.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1 RC4 Release Notes -aliases: ['/docs/dev/releases/release-2.1-rc.4/','/docs/dev/releases/21rc4/'] +summary: TiDB 2.1 RC4 was released on October 23, 2018, with improvements in stability, SQL optimizer, statistics information, and execution engine. Fixes include issues with SQL optimizer, execution engine, statistics, server, compatibility, expressions, and DDL. PD fixes issues with tombstone TiKV, data race, PD server getting stuck, and leader switching. TiKV optimizes RocksDB Write stall issue, adds raftstore tick metrics, and upgrades RocksDB and grpcio. --- # TiDB 2.1 RC4 Release Notes diff --git a/releases/release-2.1-rc.5.md b/releases/release-2.1-rc.5.md index 85a56f15fea5a..3886d20f3f717 100644 --- a/releases/release-2.1-rc.5.md +++ b/releases/release-2.1-rc.5.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1 RC5 Release Notes -aliases: ['/docs/dev/releases/release-2.1-rc.5/','/docs/dev/releases/21rc5/'] +summary: TiDB 2.1 RC5 was released on November 12, 2018, with improvements in stability, SQL optimizer, statistics, and execution engine. Fixes include issues with IndexReader, IndexScan Prepared statement, Union statement, and JSON data conversion. Server improvements include log readability, table data retrieval, and environment variable additions. PD fixes issues related to Region key reading, `regions/check` API, PD restart join, and event loss. TiKV improves error messages, adds panic mark file, downgrades grpcio, and adds an upper limit to the `kv_scan` interface. Tools now support the TiDB-Binlog cluster. --- diff --git a/releases/release-2.1.1.md b/releases/release-2.1.1.md index 39aa893d95abe..ef7d195c2f55f 100644 --- a/releases/release-2.1.1.md +++ b/releases/release-2.1.1.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1.1 Release Notes -aliases: ['/docs/dev/releases/release-2.1.1/','/docs/dev/releases/2.1.1/'] +summary: TiDB 2.1.1 was released on December 12, 2018, with improvements in stability, SQL optimizer, statistics information, and execution engine. Fixes include round error of negative date, uncompress function data length check, and transaction retries. Default character set and collation of tables changed to utf8mb4. PD and TiKV also received various fixes and optimizations. Lightning tool optimized analyze mechanism and added support for storing checkpoint information locally. TiDB Binlog fixed output bug of pb files for tables with only primary key column. --- # TiDB 2.1.1 Release Notes diff --git a/releases/release-2.1.10.md b/releases/release-2.1.10.md index 6c4a611915bf4..c37726248bbf8 100644 --- a/releases/release-2.1.10.md +++ b/releases/release-2.1.10.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1.10 Release Notes -aliases: ['/docs/dev/releases/release-2.1.10/','/docs/dev/releases/2.1.10/'] +summary: TiDB 2.1.10 was released on May 22, 2019, with various bug fixes and improvements. The release includes fixes for issues related to table schema, read results, generated columns, datetime functions, slow logs, and more. Additionally, improvements were made to TiKV and tools like TiDB Lightning and TiDB Binlog. The TiDB Ansible version 2.1.10 also received updates. --- # TiDB 2.1.10 Release Notes diff --git a/releases/release-2.1.11.md b/releases/release-2.1.11.md index a94cb39cd44d5..77984c4f5d4a8 100644 --- a/releases/release-2.1.11.md +++ b/releases/release-2.1.11.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1.11 Release Notes -aliases: ['/docs/dev/releases/release-2.1.11/','/docs/dev/releases/2.1.11/'] +summary: TiDB 2.1.11 was released on June 03, 2019. It includes fixes for various issues in TiDB, PD, TiKV, and Tools. Some highlights are the fix for incorrect schema in delete from join, calculation errors of unix_timestamp(), and the addition of Drainer parameters in TiDB Ansible. --- # TiDB 2.1.11 Release Notes diff --git a/releases/release-2.1.12.md b/releases/release-2.1.12.md index 9f5b97a691a11..34510d7f1bd72 100644 --- a/releases/release-2.1.12.md +++ b/releases/release-2.1.12.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1.12 Release Notes -aliases: ['/docs/dev/releases/release-2.1.12/','/docs/dev/releases/2.1.12/'] +summary: TiDB 2.1.12 was released on June 13, 2019. It includes various bug fixes and improvements, such as fixing issues with data type mismatches, charset altering, and GRANT operations. The release also improves compatibility with MySQL and addresses issues with functions, data conversion, and error reporting. Additionally, PD and TiKV have also been updated to fix issues related to leader election and data availability during leader transfer and power failure. --- # TiDB 2.1.12 Release Notes diff --git a/releases/release-2.1.13.md b/releases/release-2.1.13.md index 5b280f196138f..10ff3b4629341 100644 --- a/releases/release-2.1.13.md +++ b/releases/release-2.1.13.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1.13 Release Notes -aliases: ['/docs/dev/releases/release-2.1.13/','/docs/dev/releases/2.1.13/'] +summary: TiDB 2.1.13 was released on June 21, 2019. It includes features to scatter row IDs, optimize DDL metadata lifetime, fix OOM issue, update statistics, support Region presplit, improve MySQL compatibility, and fix estimation issues. TiKV fixes incomplete snapshots and adds a feature to check the validity of the block-size configuration. TiDB Binlog fixes wrong offset and adds advertise-addr configuration in Drainer. --- # TiDB 2.1.13 Release Notes diff --git a/releases/release-2.1.14.md b/releases/release-2.1.14.md index 77cd5d851ea5e..c992ff3b577d8 100644 --- a/releases/release-2.1.14.md +++ b/releases/release-2.1.14.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1.14 Release Notes -aliases: ['/docs/dev/releases/release-2.1.14/','/docs/dev/releases/2.1.14/'] +summary: TiDB 2.1.14 was released on July 04, 2019. It includes various bug fixes and improvements, such as fixing wrong query results, adding new system variables, optimizing memory usage, and adding new configuration items for TiDB Binlog and TiDB Ansible. Additionally, there are optimizations for TiKV and PD. --- # TiDB 2.1.14 Release Notes diff --git a/releases/release-2.1.15.md b/releases/release-2.1.15.md index 80e4bbd26dd6c..ff4f5286eee57 100644 --- a/releases/release-2.1.15.md +++ b/releases/release-2.1.15.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1.15 Release Notes -aliases: ['/docs/dev/releases/release-2.1.15/','/docs/dev/releases/2.1.15/'] +summary: TiDB 2.1.15 was released on July 16, 2019. It includes various bug fixes and improvements, such as fixing issues with functions like DATE_ADD and INSERT, adding new SQL statements like SHOW TABLE REGIONS, and enhancing the Audit plugin. TiKV and PD also received updates to unify log formats and improve accuracy. Additionally, there were optimizations made to TiDB Binlog and TiDB Lightning, and new monitoring items added to TiDB Ansible. --- # TiDB 2.1.15 Release Notes diff --git a/releases/release-2.1.16.md b/releases/release-2.1.16.md index dae0acee1d59b..81b5a542cc722 100644 --- a/releases/release-2.1.16.md +++ b/releases/release-2.1.16.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1.16 Release Notes -aliases: ['/docs/dev/releases/release-2.1.16/','/docs/dev/releases/2.1.16/'] +summary: TiDB 2.1.16 was released on August 15, 2019. It includes various fixes and improvements to the SQL optimizer, SQL execution engine, server, DDL, TiKV, TiDB Binlog, TiDB Lightning, and TiDB Ansible. Some notable changes include support for subqueries within SHOW statements, fixing issues with DATE_ADD function, and adding configuration items in Drainer for TiDB Binlog. --- # TiDB 2.1.16 Release Notes diff --git a/releases/release-2.1.17.md b/releases/release-2.1.17.md index d8dd5b8850e0f..2dd51533173fb 100644 --- a/releases/release-2.1.17.md +++ b/releases/release-2.1.17.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1.17 Release Notes -aliases: ['/docs/dev/releases/release-2.1.17/','/docs/dev/releases/2.1.17/'] +summary: "TiDB 2.1.17 Release Notes: New features include `WHERE` clause in `SHOW TABLE REGIONS`, `config-check` feature in TiKV and PD, `remove-tombstone` command in pd-ctl, and `worker-count` and `txn-batch` configuration items in Reparo. Improvements in PD’s scheduling process and TiKV’s starting process. Changed behaviors in TiDB slow query logs and configuration files. Fixes and optimizations in SQL Optimizer, SQL Execution Engine, Server, DDL, Monitor, TiKV, PD, TiDB Binlog, TiDB Lightning, and TiDB Ansible." --- # TiDB 2.1.17 Release Notes @@ -45,7 +45,7 @@ TiDB Ansible version: 2.1.17 - Change `start ts` recorded in slow query logs from the last retry time to the first execution time when retrying TiDB transactions [#11878](https://github.com/pingcap/tidb/pull/11878) - Add the number of keys of a transaction in `LockResolver` to avoid the scan operation on the whole Region and reduce costs of resolving locking when the number of keys is reduced [#11889](https://github.com/pingcap/tidb/pull/11889) - Fix the issue that the `succ` field value might be incorrect in slow query logs [#11886](https://github.com/pingcap/tidb/pull/11886) - - Replace the `Index_ids` filed in slow query logs with the `Index_names` field to improve the usability of slow query logs [#12063](https://github.com/pingcap/tidb/pull/12063) + - Replace the `Index_ids` field in slow query logs with the `Index_names` field to improve the usability of slow query logs [#12063](https://github.com/pingcap/tidb/pull/12063) - Fix the connection break issue caused by TiDB parsing `-` into EOF Error when `Duration` contains `-` (like `select time(‘--’)`) [#11910](https://github.com/pingcap/tidb/pull/11910) - Remove an invalid Region from `RegionCache` more quickly to reduce the number of requests sent to this Region [#11931](https://github.com/pingcap/tidb/pull/11931) - Fix the connection break issue caused by incorrectly handling the OOM panic issue when `oom-action = "cancel"` and OOM occurs in the `Insert Into … Select` syntax [#12126](https://github.com/pingcap/tidb/pull/12126) diff --git a/releases/release-2.1.18.md b/releases/release-2.1.18.md index 4006ebc2341f7..32b962dfe60aa 100644 --- a/releases/release-2.1.18.md +++ b/releases/release-2.1.18.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1.18 Release Notes -aliases: ['/docs/dev/releases/release-2.1.18/','/docs/dev/releases/2.1.18/'] +summary: TiDB 2.1.18 was released on November 4, 2019. The release includes various fixes and optimizations for SQL optimizer, SQL engine, server, DDL, monitor, and tools. Some notable improvements include support for using parameters in ORDER BY, GROUP BY, and LIMIT OFFSET, and adding new metrics for monitoring Add Index operation progress. The TiDB Ansible version 2.1.18 also includes updates and new monitoring items for TiDB Binlog. --- # TiDB 2.1.18 Release Notes diff --git a/releases/release-2.1.19.md b/releases/release-2.1.19.md index e1858c4d2bedf..08fdcb8ad4a8b 100644 --- a/releases/release-2.1.19.md +++ b/releases/release-2.1.19.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1.19 Release Notes -aliases: ['/docs/dev/releases/release-2.1.19/','/docs/dev/releases/2.1.19/'] +summary: TiDB 2.1.19 was released on December 27, 2019. It includes various fixes and optimizations for SQL optimizer, SQL execution engine, server, DDL, TiKV, PD, and TiDB Ansible. Some notable fixes include resolving incorrect query results, memory overhead reduction, and fixing issues related to timezone, data duplication, and panic occurrences. The release also includes upgrades and optimizations for TiDB Binlog and TiDB Ansible. --- # TiDB 2.1.19 Release Notes diff --git a/releases/release-2.1.2.md b/releases/release-2.1.2.md index 810feece1c73b..41014e1feabef 100644 --- a/releases/release-2.1.2.md +++ b/releases/release-2.1.2.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1.2 Release Notes -aliases: ['/docs/dev/releases/release-2.1.2/','/docs/dev/releases/2.1.2/'] +summary: TiDB 2.1.2 and TiDB Ansible 2.1.2 were released on December 22, 2018. The release includes improvements in system compatibility and stability. Key updates include compatibility with TiDB Binlog of the Kafka version, improved exit mechanism during rolling updates, and fixes for various issues. PD and TiKV also received updates, such as fixing Region merge issues and support for configuration format in the unit of 'DAY'. Additionally, TiDB Lightning and TiDB Binlog were updated to support new features and eliminate bottlenecks. --- # TiDB 2.1.2 Release Notes diff --git a/releases/release-2.1.3.md b/releases/release-2.1.3.md index ebaa726af7e4c..21f8529051d2d 100644 --- a/releases/release-2.1.3.md +++ b/releases/release-2.1.3.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1.3 Release Notes -aliases: ['/docs/dev/releases/release-2.1.3/','/docs/dev/releases/2.1.3/'] +summary: TiDB 2.1.3 and TiDB Ansible 2.1.3 are released with improvements in system stability, SQL optimizer, statistics, and execution engine. Fixes include issues with Prepared Plan Cache, Range computing, `CAST(str AS TIME(N))`, Generated Column, statistics histogram, `Sort Merge Join`, and more. Other improvements include support for Range for `_tidb_rowid` construction queries, `ALLOW_INVALID_DATES` SQL mode, and more. PD and TiKV also have fixes and improvements. TiDB Binlog fixes issues with the Pump client log and data inconsistency caused by unique key containing NULL value. --- # TiDB 2.1.3 Release Notes diff --git a/releases/release-2.1.4.md b/releases/release-2.1.4.md index ff327430aa08a..e6933eb15a57d 100644 --- a/releases/release-2.1.4.md +++ b/releases/release-2.1.4.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1.4 Release Notes -aliases: ['/docs/dev/releases/release-2.1.4/','/docs/dev/releases/2.1.4/'] +summary: TiDB 2.1.4 and TiDB Ansible 2.1.4 were released on February 15, 2019. The release includes improvements in stability, SQL optimizer, statistics, and execution engine. Fixes include issues with the SQL optimizer/executor, server, DDL, and TiKV. Lightning tool optimizations include memory usage, chunk separation removal, I/O concurrency limitation, batch data import support, and auto compactions in TiKV import mode. Additionally, support for disabling TiKV periodic Level-1 compaction parameter and limiting the number of import engines is added. Sync-diff-inspector now supports splitting chunks using TiDB statistics. --- # TiDB 2.1.4 Release Notes diff --git a/releases/release-2.1.5.md b/releases/release-2.1.5.md index df051fb25b197..f4319bb9744eb 100644 --- a/releases/release-2.1.5.md +++ b/releases/release-2.1.5.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1.5 Release Notes -aliases: ['/docs/dev/releases/release-2.1.5/','/docs/dev/releases/2.1.5/'] +summary: TiDB 2.1.5 and TiDB Ansible 2.1.5 are released on February 28, 2019. The release improves stability, SQL optimizer, statistics, and execution engine. Fixes include issues with sorting, data overflow, and SQL query results. New features include system variables, HTTP API, and detailed error messages. PD now has an option to exclude Tombstone stores, and TiKV fixes issues with data import, errors, and panic caused by Region merge. Tools like Lightning and TiDB Binlog also receive updates. --- # TiDB 2.1.5 Release Notes diff --git a/releases/release-2.1.6.md b/releases/release-2.1.6.md index fecb5b89a9d8c..e93985930ab38 100644 --- a/releases/release-2.1.6.md +++ b/releases/release-2.1.6.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1.6 Release Notes -aliases: ['/docs/dev/releases/release-2.1.6/','/docs/dev/releases/2.1.6/'] +summary: TiDB 2.1.6 and TiDB Ansible 2.1.6 were released on March 15, 2019. The release includes improvements in stability, SQL optimizer, statistics, and execution engine. Fixes and enhancements were made in SQL optimizer/executor, server, DDL, TiKV, and Tools. Notable changes include support for log_bin variable, sanity check for transactions, and fixing import failure due to non-alphanumeric characters in schema names. --- # TiDB 2.1.6 Release Notes diff --git a/releases/release-2.1.7.md b/releases/release-2.1.7.md index 4bca1a35b67ee..b49c0b711f5db 100644 --- a/releases/release-2.1.7.md +++ b/releases/release-2.1.7.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1.7 Release Notes -aliases: ['/docs/dev/releases/release-2.1.7/','/docs/dev/releases/2.1.7/'] +summary: TiDB 2.1.7 was released on March 28, 2019. It includes various bug fixes, compatibility improvements, and new features such as support for subqueries in the `DO` statement, plugin framework, and checking binlog and Pump/Drainer status using SQL statements. PD also fixed an issue related to transferring leader step in balance-region. Additionally, the default retention time of Prometheus monitoring data in TiDB Ansible has been changed to 30d. --- # TiDB 2.1.7 Release Notes diff --git a/releases/release-2.1.8.md b/releases/release-2.1.8.md index 62519fbafc9d3..cc2f8921636d8 100644 --- a/releases/release-2.1.8.md +++ b/releases/release-2.1.8.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1.8 Release Notes -aliases: ['/docs/dev/releases/release-2.1.8/','/docs/dev/releases/2.1.8/'] +summary: TiDB 2.1.8 was released on April 12, 2019. It includes various bug fixes and improvements for TiDB, PD, TiKV, Tools, and TiDB Ansible. Some notable fixes include compatibility issues with MySQL, inaccurate statistics estimation, and performance improvements. The release also adds new configuration items and features for TiDB Binlog Pump and Drainer. Additionally, TiDB Ansible now has version limits for the operating system and rolling updates. --- # TiDB 2.1.8 Release Notes diff --git a/releases/release-2.1.9.md b/releases/release-2.1.9.md index 04d815cd9888f..9cdd2d125de9e 100644 --- a/releases/release-2.1.9.md +++ b/releases/release-2.1.9.md @@ -1,6 +1,6 @@ --- title: TiDB 2.1.9 Release Notes -aliases: ['/docs/dev/releases/release-2.1.9/','/docs/dev/releases/2.1.9/'] +summary: TiDB 2.1.9 was released on May 6, 2019. It includes various bug fixes and improvements, such as fixing compatibility issues, privilege check problems, and wrong result issues. The release also includes improvements to slow query logs and support for controlling the number of rows returned by operators. Additionally, there are updates to PD, TiKV, TiDB Binlog, TiDB Lightning, and sync-diff-inspector. TiDB Ansible has also been updated with documentation links and parameter removal. --- # TiDB 2.1.9 Release Notes diff --git a/releases/release-3.0-beta.md b/releases/release-3.0-beta.md index f17d7cce3b4cb..905cc0844f005 100644 --- a/releases/release-3.0-beta.md +++ b/releases/release-3.0-beta.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0 Beta Release Notes -aliases: ['/docs/dev/releases/release-3.0-beta/','/docs/dev/releases/3.0beta/'] +summary: TiDB 3.0 Beta, released on January 19, 2019, focuses on stability, SQL optimizer, statistics, and execution engine. New features include support for views, window functions, range partitioning, and hash partitioning. The SQL optimizer has been enhanced with various optimizations, including support for index join in transactions, constant propagation optimization, and support for subqueries in the DO statement. The SQL executor has also been optimized for better performance. Privilege management, server, compatibility, and DDL have all been improved. TiDB Lightning now supports batch import for a single table, while PD and TiKV have also received various enhancements and new features. --- # TiDB 3.0 Beta Release Notes diff --git a/releases/release-3.0-ga.md b/releases/release-3.0-ga.md index 54edd52772eed..4d79ebd9d61e5 100644 --- a/releases/release-3.0-ga.md +++ b/releases/release-3.0-ga.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0 GA Release Notes -aliases: ['/docs/dev/releases/release-3.0-ga/','/docs/dev/releases/3.0-ga/'] +summary: TiDB 3.0 GA was released on June 28, 2019, with improved stability, usability, and performance. New features include Window Functions, Views, partitioned tables, and the plugin framework. The SQL Optimizer has been optimized for better performance, and DDL now supports fast recovery of mistakenly deleted tables. TiKV now supports distributed GC, multi-thread Raftstore, and batch receiving and sending Raft messages. Tools like TiDB Lightning and TiDB Binlog have also been enhanced with new features and performance improvements. The TiDB Ansible has been upgraded to support deployment and operations for TiDB Lightning, and to optimize monitoring components. --- # TiDB 3.0 GA Release Notes diff --git a/releases/release-3.0.0-beta.1.md b/releases/release-3.0.0-beta.1.md index c97e4d70e4c32..73eac2dd484f7 100644 --- a/releases/release-3.0.0-beta.1.md +++ b/releases/release-3.0.0-beta.1.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0.0 Beta.1 Release Notes -aliases: ['/docs/dev/releases/release-3.0.0-beta.1/','/docs/dev/releases/3.0.0-beta.1/'] +summary: TiDB 3.0.0 Beta.1 was released on March 26, 2019, with improved stability, usability, features, SQL optimizer, statistics, and execution engine. The release includes support for various SQL functions, privilege management, server enhancements, DDL improvements, and PD and TiKV optimizations. Tools like TiDB Binlog, Lightning, and data replication comparison tool have also been updated with new features and improvements. --- # TiDB 3.0.0 Beta.1 Release Notes diff --git a/releases/release-3.0.0-rc.1.md b/releases/release-3.0.0-rc.1.md index 281a5e982d7f5..036c6bb940d06 100644 --- a/releases/release-3.0.0-rc.1.md +++ b/releases/release-3.0.0-rc.1.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0.0-rc.1 Release Notes -aliases: ['/docs/dev/releases/release-3.0.0-rc.1/','/docs/dev/releases/3.0.0-rc.1/'] +summary: TiDB 3.0.0-rc.1 was released on May 10, 2019, with improved stability, usability, features, SQL optimizer, statistics, and execution engine. The release includes enhancements to the SQL optimizer, execution engine, server, DDL, PD, TiKV, TiDB Binlog, Lightning, sync-diff-inspector, and TiDB Ansible. Notable improvements include support for SQL Plan Management, memory usage tracking, and control in the execution engine, and the addition of the `pre_split_regions` option for `CREATE TABLE` statements in DDL. The release also includes various bug fixes and performance optimizations. --- # TiDB 3.0.0-rc.1 Release Notes diff --git a/releases/release-3.0.0-rc.2.md b/releases/release-3.0.0-rc.2.md index d8aa5737cebc4..dd1a79cd9c7c5 100644 --- a/releases/release-3.0.0-rc.2.md +++ b/releases/release-3.0.0-rc.2.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0.0-rc.2 Release Notes -aliases: ['/docs/dev/releases/release-3.0.0-rc.2/','/docs/dev/releases/3.0.0-rc.2/'] +summary: TiDB 3.0.0-rc.2 was released on May 28, 2019, with improvements in stability, usability, features, SQL optimizer, statistics, and execution engine. The release includes enhancements to the SQL optimizer, execution engine, server, DDL, PD, TiKV, and tools like TiDB Binlog and TiDB Lightning. Some notable improvements include support for Index Join in more scenarios, handling virtual columns properly, and adding a metric to track data replication downstream. --- # TiDB 3.0.0-rc.2 Release Notes diff --git a/releases/release-3.0.0-rc.3.md b/releases/release-3.0.0-rc.3.md index fe2040a8cbf0d..cf042b59a6d4e 100644 --- a/releases/release-3.0.0-rc.3.md +++ b/releases/release-3.0.0-rc.3.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0.0-rc.3 Release Notes -aliases: ['/docs/dev/releases/release-3.0.0-rc.3/','/docs/dev/releases/3.0.0-rc.3/'] +summary: TiDB 3.0.0-rc.3 was released on June 21, 2019, with improvements in stability, usability, features, SQL optimizer, statistics, and execution engine. Fixes and new features were added to TiDB, PD, TiKV, and TiDB Ansible. Notable improvements include automatic loading statistics, manual splitting of table and index regions, and support for pessimistic transactions in TiKV. --- # TiDB 3.0.0-rc.3 Release Notes diff --git a/releases/release-3.0.1.md b/releases/release-3.0.1.md index 790b6f4895859..838516404928e 100644 --- a/releases/release-3.0.1.md +++ b/releases/release-3.0.1.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0.1 Release Notes -aliases: ['/docs/dev/releases/release-3.0.1/','/docs/dev/releases/3.0.1/'] +summary: "TiDB 3.0.1 Release Notes July 16, 2019. TiDB version 3.0.1. Add support for MAX_EXECUTION_TIME feature. Support auto-adjustment of incremental gap for auto-increment IDs. Add ADMIN PLUGINS ENABLE/DISABLE SQL statement. Prohibit Window Functions from being cached in Prepare Plan Cache. Fix various bugs and issues. TiKV: Add statistics of blob file size. Fix core dump issue. PD: Add enable-grpc-gateway configuration option. Optimize hot Region scheduling strategy. Tools: TiDB Binlog - Optimize Pump GC strategy. TiDB Lightning - Fix import error. TiDB Ansible - Add precheck feature, update monitoring information." --- # TiDB 3.0.1 Release Notes diff --git a/releases/release-3.0.10.md b/releases/release-3.0.10.md index 3b93e674ac338..2a8139e1489e6 100644 --- a/releases/release-3.0.10.md +++ b/releases/release-3.0.10.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0.10 Release Notes -aliases: ['/docs/dev/releases/release-3.0.10/','/docs/dev/releases/3.0.10/'] +summary: TiDB 3.0.10 was released on February 20, 2020. It includes various bug fixes and improvements for TiDB, TiKV, PD, and TiDB Ansible. Some notable fixes include wrong Join results, data visibility issues, and system panic problems. TiDB Ansible also added new monitoring items to the dashboard. The release notes recommend using the latest 3.0.x version due to known issues in this release. --- # TiDB 3.0.10 Release Notes @@ -51,7 +51,7 @@ TiDB Ansible version: 3.0.10 + Raftstore - Fix the system panic issue #6460 or data loss issue #598 caused by Region merge failure [#6481](https://github.com/tikv/tikv/pull/6481) - - Support `yield` to optimize scheduling fairness, and support pre-transfering the leader to improve leader scheduling stability [#6563](https://github.com/tikv/tikv/pull/6563) + - Support `yield` to optimize scheduling fairness, and support pre-transferring the leader to improve leader scheduling stability [#6563](https://github.com/tikv/tikv/pull/6563) ## PD diff --git a/releases/release-3.0.11.md b/releases/release-3.0.11.md index cbe6a80803e34..77d470237582e 100644 --- a/releases/release-3.0.11.md +++ b/releases/release-3.0.11.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0.11 Release Notes -aliases: ['/docs/dev/releases/release-3.0.11/','/docs/dev/releases/3.0.11/'] +summary: TiDB 3.0.11 was released on March 4, 2020. It includes compatibility changes, new features, bug fixes, and updates for TiDB, TiDB Binlog, TiDB Lightning, TiKV, and TiDB Ansible. Some known issues are fixed in new versions, so it is recommended to use the latest 3.0.x version. --- # TiDB 3.0.11 Release Notes diff --git a/releases/release-3.0.12.md b/releases/release-3.0.12.md index c60f5d1bfe451..8c1953f7c89be 100644 --- a/releases/release-3.0.12.md +++ b/releases/release-3.0.12.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0.12 Release Notes -aliases: ['/docs/dev/releases/release-3.0.12/','/docs/dev/releases/3.0.12/'] +summary: TiDB 3.0.12 was released on March 16, 2020. It includes compatibility changes, new features, bug fixes, and improvements for TiDB, TiKV, PD, and TiDB Ansible. Some known issues are fixed in new versions, so it is recommended to use the latest 3.0.x version. New features include dynamic loading of replaced certificate files, flow limiting for DDL requests, and support for exiting the TiDB server when binlog write fails. Bug fixes address issues with locking, error message display, decimal point accuracy, and data index inconsistency. Additionally, improvements have been made to TiKV's flow control mechanism and PD's Region information processing. --- # TiDB 3.0.12 Release Notes diff --git a/releases/release-3.0.13.md b/releases/release-3.0.13.md index 1622493ec1eeb..befdb9c0e6094 100644 --- a/releases/release-3.0.13.md +++ b/releases/release-3.0.13.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0.13 Release Notes -aliases: ['/docs/dev/releases/release-3.0.13/','/docs/dev/releases/3.0.13/'] +summary: TiDB 3.0.13 was released on April 22, 2020. The bug fixes include resolving issues with the `INSERT ... ON DUPLICATE KEY UPDATE` statement and fixing the system getting stuck and becoming unavailable during `Region Merge` in TiKV. --- # TiDB 3.0.13 Release Notes diff --git a/releases/release-3.0.14.md b/releases/release-3.0.14.md index 7fe7fa8ae9a66..58258b91dbb6d 100644 --- a/releases/release-3.0.14.md +++ b/releases/release-3.0.14.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0.14 Release Notes -aliases: ['/docs/dev/releases/release-3.0.14/','/docs/dev/releases/3.0.14/'] +summary: TiDB 3.0.14 was released on May 9, 2020. The release includes compatibility changes, important bug fixes, new features, bug fixes, and improvements for TiDB, TiKV, and Tools. Some of the bug fixes include issues with query results, panic occurrences, and incorrect behavior. New features include enhanced syntax support and improved performance. --- # TiDB 3.0.14 Release Notes diff --git a/releases/release-3.0.15.md b/releases/release-3.0.15.md index 87501606431cf..0ba0685dda006 100644 --- a/releases/release-3.0.15.md +++ b/releases/release-3.0.15.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0.15 Release Notes -aliases: ['/docs/dev/releases/release-3.0.15/'] +summary: TiDB 3.0.15 was released on June 5, 2020. New features include support for admin recover index and admin check index statements on partitioned tables, as well as optimization of memory allocation mechanism. Bug fixes address issues such as incorrect results in PointGet and inconsistent results between TiDB and MySQL when XOR operates on a floating-point number. TiKV fixes issues related to memory defragmentation and gRPC disconnection. --- # TiDB 3.0.15 Release Notes diff --git a/releases/release-3.0.16.md b/releases/release-3.0.16.md index 0f5586e6fffe3..a58c0e37a550a 100644 --- a/releases/release-3.0.16.md +++ b/releases/release-3.0.16.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0.16 Release Notes -aliases: ['/docs/dev/releases/release-3.0.16/'] +summary: TiDB 3.0.16 was released on July 03, 2020. The release includes improvements such as support for 'is null' filter condition, handling of SQL timeout issues, and removal of sensitive information in slow query logs. Bug fixes include resolving data inconsistency issues, fixing panic issues, and addressing errors in JSON comparison and query results. TiKV and PD also received bug fixes for issues related to store heartbeats, peer removal, and error handling. --- # TiDB 3.0.16 Release Notes diff --git a/releases/release-3.0.17.md b/releases/release-3.0.17.md index 4bf60894e8c02..37219785901ab 100644 --- a/releases/release-3.0.17.md +++ b/releases/release-3.0.17.md @@ -1,5 +1,6 @@ --- title: TiDB 3.0.17 Release Notes +summary: TiDB 3.0.17 was released on Aug 3, 2020. The release includes improvements such as decreasing the default value of the query-feedback-limit configuration item and bug fixes like returning the actual error message instead of an empty set. TiKV also added the hibernate-timeout configuration to improve rolling update performance. TiDB Lightning deprecated the black-white-list filter format and fixed the issue of the log-file flag being ignored. --- # TiDB 3.0.17 Release Notes diff --git a/releases/release-3.0.18.md b/releases/release-3.0.18.md index c592505d34935..82a249f96ab8a 100644 --- a/releases/release-3.0.18.md +++ b/releases/release-3.0.18.md @@ -1,5 +1,6 @@ --- title: TiDB 3.0.18 Release Notes +summary: TiDB 3.0.18 was released on August 21, 2020. The release includes improvements to TiDB Binlog and bug fixes for TiDB and TiKV. Bug fixes for TiDB include issues with handling decimal, set, and enum types, as well as problems with duplicate keys and cached execution plans. TiKV's bug fix involves changing the GC failure log level. TiDB Lightning also received fixes for issues with the log file argument, syntax errors, and unexpected calls. --- # TiDB 3.0.18 Release Notes diff --git a/releases/release-3.0.19.md b/releases/release-3.0.19.md index c66a7e60cb448..56aa1d4100b6c 100644 --- a/releases/release-3.0.19.md +++ b/releases/release-3.0.19.md @@ -1,5 +1,6 @@ --- title: TiDB 3.0.19 Release Notes +summary: TiDB 3.0.19 was released on September 25, 2020. Compatibility changes include import path and copyright information updates. Improvements were made to mitigate failure recovery impact, support concurrency adjustment, and set nonadjustable values. Bug fixes were made for query errors, privilege checks, type changes, constraint checks, table lock release, operator handling, and panic parsing. Tools like TiDB Lightning also received fixes for process exit timing. --- # TiDB 3.0.19 Release Notes diff --git a/releases/release-3.0.2.md b/releases/release-3.0.2.md index 17ffa6685150a..f8508ff3eb612 100644 --- a/releases/release-3.0.2.md +++ b/releases/release-3.0.2.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0.2 Release Notes -aliases: ['/docs/dev/releases/release-3.0.2/','/docs/dev/releases/3.0.2/'] +summary: TiDB 3.0.2 was released on August 7, 2019, with various fixes and improvements. The release includes fixes for SQL optimizer, SQL execution engine, server, DDL, monitor, TiKV, PD, TiDB Binlog, TiDB Lightning, and TiDB Ansible. Fixes include issues with query plans, query results, error messages, and performance optimizations. --- # TiDB 3.0.2 Release Notes diff --git a/releases/release-3.0.20.md b/releases/release-3.0.20.md index 0ad78bd9b77b4..6d0c2daca8fca 100644 --- a/releases/release-3.0.20.md +++ b/releases/release-3.0.20.md @@ -1,5 +1,6 @@ --- title: TiDB 3.0.20 Release Notes +summary: TiDB 3.0.20 was released on December 25, 2020. The release includes compatibility changes, improvements, and bug fixes for TiDB, TiKV, and PD. Some notable bug fixes include addressing issues with incorrect cache of transaction status, inaccurate statistics, and stack overflow. --- # TiDB 3.0.20 Release Notes diff --git a/releases/release-3.0.3.md b/releases/release-3.0.3.md index 0d59196f71cb2..2381683ccf9b6 100644 --- a/releases/release-3.0.3.md +++ b/releases/release-3.0.3.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0.3 Release Notes -aliases: ['/docs/dev/releases/release-3.0.3/','/docs/dev/releases/3.0.3/'] +summary: TiDB 3.0.3 was released on August 29, 2019. It includes various fixes and updates for SQL optimizer, SQL execution engine, server, DDL, monitor, TiKV, PD, TiDB Binlog, TiDB Lightning, and TiDB Ansible. Notable fixes include issues with incorrect results, type errors, panic occurrences, and permission check errors. The release also optimizes PD operations, removes unsupported Grafana Collector components, and updates TiKV alerting rules. Additionally, TiDB Ansible now supports Spark V2.4.3 and TiSpark V2.1.4. --- # TiDB 3.0.3 Release Notes diff --git a/releases/release-3.0.4.md b/releases/release-3.0.4.md index 0837d4a7eaa80..b2a703111331e 100644 --- a/releases/release-3.0.4.md +++ b/releases/release-3.0.4.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0.4 Release Notes -aliases: ['/docs/dev/releases/release-3.0.4/','/docs/dev/releases/3.0.4/'] +summary: TiDB 3.0.4 was released on October 8, 2019, with new features including system table for troubleshooting SQL performance issues, improvements in split performance and reverse scan, and fixed issues related to slow query logs and data replication. The release also includes contributions from the community and updates to TiDB, TiKV, PD, and TiDB Ansible. --- # TiDB 3.0.4 Release Notes @@ -42,7 +42,7 @@ TiDB Ansible version: 3.0.4 ## TiDB - SQL Optimizer - - Fix the issue that invalid query ranges might be resulted when splitted by feedback [#12170](https://github.com/pingcap/tidb/pull/12170) + - Fix the issue that invalid query ranges might be resulted when split by feedback [#12170](https://github.com/pingcap/tidb/pull/12170) - Display the returned error of the `SHOW STATS_BUCKETS` statement in hexadecimal rather than return errors when the result contains invalid Keys [#12094](https://github.com/pingcap/tidb/pull/12094) - Fix the issue that when a query contains the `SLEEP` function (for example, `select 1 from (select sleep(1)) t;)`), column pruning causes invalid `sleep(1)` during query [#11953](https://github.com/pingcap/tidb/pull/11953) - Use index scan to lower IO when a query only concerns the number of columns rather than the table data [#12112](https://github.com/pingcap/tidb/pull/12112) diff --git a/releases/release-3.0.5.md b/releases/release-3.0.5.md index 2a8474be3bf61..5aaf5d428d4f3 100644 --- a/releases/release-3.0.5.md +++ b/releases/release-3.0.5.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0.5 Release Notes -aliases: ['/docs/dev/releases/release-3.0.5/','/docs/dev/releases/3.0.5/'] +summary: TiDB 3.0.5 was released on October 25, 2019, with various improvements and bug fixes. The release includes enhancements to the SQL optimizer, SQL execution engine, server, DDL, monitor, TiKV, PD, TiDB Binlog, TiDB Lightning, and TiDB Ansible. Improvements include support for boundary checking on Window Functions, fixing issues with index join and outer join, and adding monitoring metrics for various operations. Additionally, TiKV received storage and performance optimizations, while PD saw improvements in storage precision and HTTP request handling. TiDB Ansible also received updates to monitoring metrics and configuration file simplification. --- # TiDB 3.0.5 Release Notes diff --git a/releases/release-3.0.6.md b/releases/release-3.0.6.md index f38ce5cddfda8..9788e9361b928 100644 --- a/releases/release-3.0.6.md +++ b/releases/release-3.0.6.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0.6 Release Notes -aliases: ['/docs/dev/releases/release-3.0.6/','/docs/dev/releases/3.0.6/'] +summary: TiDB 3.0.6 was released on November 28, 2019, with various fixes and optimizations. The release includes improvements to the SQL optimizer, SQL execution engine, server, DDL, TiKV, PD, TiDB Binlog, and TiDB Lightning. Fixes include issues with window function AST, pushing down `STREAM AGG()`, handling quotes for SQL binding, and more. TiKV improvements include accurate `lock_manager`, support for `innodb_lock_wait_timeout`, and dynamic modification of the GC I/O limit using `tikv-ctl`. PD enhancements include lower client log level and warning log for generating a timestamp. TiDB Binlog and TiDB Lightning also received fixes and improvements. --- # TiDB 3.0.6 Release Notes diff --git a/releases/release-3.0.7.md b/releases/release-3.0.7.md index 7f99c56e3907c..d3fffe56bc425 100644 --- a/releases/release-3.0.7.md +++ b/releases/release-3.0.7.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0.7 Release Notes -aliases: ['/docs/dev/releases/release-3.0.7/','/docs/dev/releases/3.0.7/'] +summary: TiDB 3.0.7 was released on December 4, 2019. It includes fixes for issues related to lock TTL, timezone parsing, result accuracy, data precision, and statistics accuracy. TiKV also received updates to improve deadlock detection and fix a memory leak issue. --- # TiDB 3.0.7 Release Notes diff --git a/releases/release-3.0.8.md b/releases/release-3.0.8.md index 966ec1e5569f7..fd50ac09707d7 100644 --- a/releases/release-3.0.8.md +++ b/releases/release-3.0.8.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0.8 Release Notes -aliases: ['/docs/dev/releases/release-3.0.8/','/docs/dev/releases/3.0.8/'] +summary: TiDB 3.0.8 was released on December 31, 2019. It includes various fixes and improvements for SQL optimizer, SQL execution engine, DDL, server, transaction, monitor, TiKV, PD, and TiDB Ansible. Notable changes include SQL binding plan fixes, error message optimizations, and support for certificate-based authentication. The default value of `tidb_txn_mode` variable is updated to `"pessimistic"`. PD also received performance optimizations and bug fixes. TiDB Ansible saw various logic optimizations and upgrades. --- # TiDB 3.0.8 Release Notes diff --git a/releases/release-3.0.9.md b/releases/release-3.0.9.md index 4ce2fafe0e7a0..6f4554f148242 100644 --- a/releases/release-3.0.9.md +++ b/releases/release-3.0.9.md @@ -1,6 +1,6 @@ --- title: TiDB 3.0.9 Release Notes -aliases: ['/docs/dev/releases/release-3.0.9/','/docs/dev/releases/3.0.9/'] +summary: TiDB 3.0.9 was released on January 14, 2020. It includes fixes for known issues and new features. Some improvements were made to Executor, Server, DDL, Planner, TiKV, PD, Tools, and TiDB Ansible. Notable changes include support for system variables, monitoring metrics, and optimizations for transaction execution latency. Additionally, support for using backlash in the location label name and automatically creating directories for TiDB Lightning deployment was added. --- # TiDB 3.0.9 Release Notes diff --git a/releases/release-3.1.0-beta.1.md b/releases/release-3.1.0-beta.1.md index f3e2c3bc48d7c..c4642b33928b9 100644 --- a/releases/release-3.1.0-beta.1.md +++ b/releases/release-3.1.0-beta.1.md @@ -1,6 +1,6 @@ --- title: TiDB 3.1 Beta.1 Release Notes -aliases: ['/docs/dev/releases/release-3.1.0-beta.1/','/docs/dev/releases/3.1.0-beta.1/'] +summary: TiDB 3.1 Beta.1 was released on January 10, 2020. The release includes changes to TiKV, such as renaming backup files and adding incremental backup features. Tools like BR have improved backup progress information and added features for partitioned tables. TiDB Ansible now automatically disables Transparent Huge Pages and adds Grafana monitoring for BR components. Overall, the release focuses on improving backup and restore processes, monitoring, and deployment optimization. --- # TiDB 3.1 Beta.1 Release Notes diff --git a/releases/release-3.1.0-beta.2.md b/releases/release-3.1.0-beta.2.md index 92a4c5a6aecce..6b477d4f9f20e 100644 --- a/releases/release-3.1.0-beta.2.md +++ b/releases/release-3.1.0-beta.2.md @@ -1,6 +1,6 @@ --- title: TiDB 3.1 Beta.2 Release Notes -aliases: ['/docs/dev/releases/release-3.1.0-beta.2/','/docs/dev/releases/3.1.0-beta.2/'] +summary: TiDB 3.1 Beta.2 was released on March 9, 2020. It includes compatibility changes, new features, bug fixes, and improvements for TiDB, TiKV, PD Client, Backup, PD, TiFlash, and TiDB Ansible. Some known issues are fixed in new versions, so it is recommended to use the latest 3.1.x version. --- # TiDB 3.1 Beta.2 Release Notes diff --git a/releases/release-3.1.0-beta.md b/releases/release-3.1.0-beta.md index 83b69a195ba28..8885c9c55d124 100644 --- a/releases/release-3.1.0-beta.md +++ b/releases/release-3.1.0-beta.md @@ -1,6 +1,6 @@ --- title: TiDB 3.1 Beta Release Notes -aliases: ['/docs/dev/releases/release-3.1.0-beta/','/docs/dev/releases/3.1.0-beta/'] +summary: TiDB 3.1 Beta was released on December 20, 2019. It includes SQL Optimizer improvements and supports the Follower Read feature. TiKV now supports distributed backup and restore, as well as the Follower Read feature. PD also supports distributed backup and restore. --- # TiDB 3.1 Beta Release Notes diff --git a/releases/release-3.1.0-ga.md b/releases/release-3.1.0-ga.md index dfc91bbe2f398..d2b15102edcf1 100644 --- a/releases/release-3.1.0-ga.md +++ b/releases/release-3.1.0-ga.md @@ -1,6 +1,6 @@ --- title: TiDB 3.1.0 GA Release Notes -aliases: ['/docs/dev/releases/release-3.1.0-ga/','/docs/dev/releases/3.1.0-ga/'] +summary: TiDB 3.1.0 GA was released on April 16, 2020. It includes compatibility changes, new features, bug fixes, and improvements for TiDB, TiFlash, TiKV, and tools like Backup & Restore and TiDB Binlog. Notable changes include support for displaying Coprocessor tasks, optimizing hot Region scheduling, and fixing various panic and data loss issues. The release also includes improvements to TiDB Ansible for better monitoring and configuration parameters. --- # TiDB 3.1.0 GA Release Notes diff --git a/releases/release-3.1.0-rc.md b/releases/release-3.1.0-rc.md index 8ec5073bf458f..2e416fce5cd12 100644 --- a/releases/release-3.1.0-rc.md +++ b/releases/release-3.1.0-rc.md @@ -1,6 +1,6 @@ --- title: TiDB 3.1 RC Release Notes -aliases: ['/docs/dev/releases/release-3.1.0-rc/','/docs/dev/releases/3.1.0-rc/'] +summary: TiDB 3.1 RC was released on April 2, 2020. It includes new features such as improved partition pruning, support for `RECOVER` syntax, and TLS certificate updates. Bug fixes include resolving issues with TiFlash replica, `last_insert_id`, and `Aggregation` pushdown. TiKV now supports TLS authentication and AWS IAM web identity for backup. PD has fixed data race issues and placement rule inconsistencies. Tools like TiDB Lightning and BR have also been optimized and fixed. --- # TiDB 3.1 RC Release Notes diff --git a/releases/release-3.1.1.md b/releases/release-3.1.1.md index 43a4f6a91408c..eb52237a00cdf 100644 --- a/releases/release-3.1.1.md +++ b/releases/release-3.1.1.md @@ -1,6 +1,6 @@ --- title: TiDB 3.1.1 Release Notes -aliases: ['/docs/dev/releases/release-3.1.1/','/docs/dev/releases/3.1.1/'] +summary: TiDB 3.1.1 was released on April 30, 2020. New features include table option for `auto_rand_base` and `Feature ID` comment. Bug fixes include isolation read setting, partition selection syntax, and wrong results from nested queries. TiFlash also received bug fixes and improvements in data reading and storage path modification. Backup & Restore (BR) fixed issues related to table restoration and data insertion. --- # TiDB 3.1.1 Release Notes diff --git a/releases/release-3.1.2.md b/releases/release-3.1.2.md index 4c3348588b755..9222682627a27 100644 --- a/releases/release-3.1.2.md +++ b/releases/release-3.1.2.md @@ -1,6 +1,6 @@ --- title: TiDB 3.1.2 Release Notes -aliases: ['/docs/dev/releases/release-3.1.2/'] +summary: TiDB 3.1.2 was released on June 4, 2020. Bug fixes include error handling during backup and restoration with S3 and GCS, and a `DefaultNotFound` error during restoration. Tools like Backup & Restore (BR) now automatically retry on poor network, fix restoration failures, data loss issues, and support AWS KMS for server-side encryption with S3 storage. --- # TiDB 3.1.2 Release Notes diff --git a/releases/release-4.0-ga.md b/releases/release-4.0-ga.md index db2f4b6f073e0..7188ad5697196 100644 --- a/releases/release-4.0-ga.md +++ b/releases/release-4.0-ga.md @@ -1,6 +1,6 @@ --- title: TiDB 4.0 GA Release Notes -aliases: ['/docs/dev/releases/release-4.0-ga/'] +summary: TiDB 4.0.0 GA was released on May 28, 2020. This version optimized error messages for large-sized transactions, improved usability of `Changefeed` configuration file, added new configuration items and support for various syntax and functions, fixed multiple bugs and issues in TiKV, TiFlash, PD, and Tools, added new monitoring items and support for various features in PD, and fixed various issues in Backup & Restore (BR) and TiCDC. --- # TiDB 4.0 GA Release Notes diff --git a/releases/release-4.0.0-beta.1.md b/releases/release-4.0.0-beta.1.md index 3b87df979d200..a9f6c438a6fb3 100644 --- a/releases/release-4.0.0-beta.1.md +++ b/releases/release-4.0.0-beta.1.md @@ -1,6 +1,6 @@ --- title: TiDB 4.0.0 Beta.1 Release Notes -aliases: ['/docs/dev/releases/release-4.0.0-beta.1/','/docs/dev/releases/4.0.0-beta.1/'] +summary: TiDB 4.0.0 Beta.1 was released on February 28, 2020. It includes compatibility changes, new features, and bug fixes. Some highlights include support for SQL performance diagnosis, the `Sequence` function, and TLS support between components. Additionally, TiDB Lightning now has a bug fix for the web interface. --- # TiDB 4.0.0 Beta.1 Release Notes @@ -83,7 +83,7 @@ TiDB Ansible version: 4.0.0-beta.1 + Fix the incorrect results of `BatchPointGet` when `plan cache` is enabled [#14855](https://github.com/pingcap/tidb/pull/14855) + Fix the issue that data is inserted into the wrong partitioned table after the timezone is modified [#14370](https://github.com/pingcap/tidb/pull/14370) + Fix the panic occurred when rebuilding expression using the invalid name of the `IsTrue` function during the outer join simplification [#14515](https://github.com/pingcap/tidb/pull/14515) - + Fix the the incorrect privilege check for the`show binding` statement [#14443](https://github.com/pingcap/tidb/pull/14443) + + Fix the incorrect privilege check for the`show binding` statement [#14443](https://github.com/pingcap/tidb/pull/14443) * TiKV + Fix the inconsistent behaviors of the `CAST` function in TiDB and TiKV [#6463](https://github.com/tikv/tikv/pull/6463) [#6461](https://github.com/tikv/tikv/pull/6461) [#6459](https://github.com/tikv/tikv/pull/6459) [#6474](https://github.com/tikv/tikv/pull/6474) [#6492](https://github.com/tikv/tikv/pull/6492) [#6569](https://github.com/tikv/tikv/pull/6569) diff --git a/releases/release-4.0.0-beta.2.md b/releases/release-4.0.0-beta.2.md index 3e08aae9c1668..b3443cf8bada5 100644 --- a/releases/release-4.0.0-beta.2.md +++ b/releases/release-4.0.0-beta.2.md @@ -1,6 +1,6 @@ --- title: TiDB 4.0.0 Beta.2 Release Notes -aliases: ['/docs/dev/releases/release-4.0.0-beta.2/','/docs/dev/releases/4.0.0-beta.2/'] +summary: TiDB 4.0.0 Beta.2 was released on March 18, 2020. The new features include support for persisting dynamically updated configurations, bidirectional data replication, TLS configuration, change data capture, and experimental features like incremental backup. Bug fixes address issues with panic, hibernate regions, replication delay, and compatibility. TiDB Ansible now supports injecting node information to etcd and deploying services on the ARM platform. --- # TiDB 4.0.0 Beta.2 Release Notes diff --git a/releases/release-4.0.0-beta.md b/releases/release-4.0.0-beta.md index faa45cc7ec4bc..90ce06fb7009c 100644 --- a/releases/release-4.0.0-beta.md +++ b/releases/release-4.0.0-beta.md @@ -1,6 +1,6 @@ --- title: TiDB 4.0 Beta Release Notes -aliases: ['/docs/dev/releases/release-4.0.0-beta/','/docs/dev/releases/4.0.0-beta/'] +summary: TiDB version 4.0.0-beta and TiDB Ansible version 4.0.0-beta were released on January 17, 2020. The release includes various improvements such as increased accuracy in calculating the cost of Index Join, support for Table Locks, and optimization of the error code of SQL error messages. TiKV was also upgraded to RocksDB version 6.4.6 and now supports quick backup and restoration. PD now supports optimizing hotspot scheduling and adding Placement Rules feature. TiDB Lightning added a parameter to set the password of the downstream database, and TiDB Ansible now supports deploying and maintaining TiFlash. --- # TiDB 4.0 Beta Release Notes diff --git a/releases/release-4.0.0-rc.1.md b/releases/release-4.0.0-rc.1.md index 465dd0052940c..ff24fc7397f44 100644 --- a/releases/release-4.0.0-rc.1.md +++ b/releases/release-4.0.0-rc.1.md @@ -1,6 +1,6 @@ --- title: TiDB 4.0 RC.1 Release Notes -aliases: ['/docs/dev/releases/release-4.0.0-rc.1/','/docs/dev/releases/4.0.0-rc.1/'] +summary: TiDB 4.0 RC.1 was released on April 28, 2020. The release includes compatibility changes, important bug fixes, new features, and bug fixes for TiKV, TiDB, TiFlash, TiCDC, Backup & Restore (BR), and Placement Driver (PD). The bug fixes address issues such as data inconsistency, deadlock, and replication failure. New features include support for sending Coprocessor requests to TiFlash in batches and enabling the load-based split region operation. Additionally, TiFlash now supports pushing down the fromUnixTime and dateFormat functions. --- # TiDB 4.0 RC.1 Release Notes diff --git a/releases/release-4.0.0-rc.2.md b/releases/release-4.0.0-rc.2.md index 7daa263e87642..5155e65e2b754 100644 --- a/releases/release-4.0.0-rc.2.md +++ b/releases/release-4.0.0-rc.2.md @@ -1,6 +1,6 @@ --- title: TiDB 4.0 RC.2 Release Notes -aliases: ['/docs/dev/releases/release-4.0.0-rc.2/'] +summary: TiDB 4.0 RC.2 was released on May 15, 2020. The release includes compatibility changes, important bug fixes, new features, and bug fixes for TiDB, TiKV, PD, TiFlash, and various tools. Some notable changes include the removal of the size limit for a single transaction when TiDB Binlog is enabled, support for the BACKUP and RESTORE commands, and the addition of encryption-related monitoring metrics in Grafana dashboard. Additionally, there are numerous bug fixes for issues such as wrong partition selection, incorrect index range building, and performance reduction. The release also introduces new features like support for the auto_random option in the CREATE TABLE statement and the ability to manage replication tasks using cdc cli. --- # TiDB 4.0 RC.2 Release Notes @@ -89,7 +89,7 @@ TiDB version: 4.0.0-rc.2 - Change the name of the Count graph of **Read Index** in Grafana to **Ops** - Optimize the data for opening file descriptors when the system load is low to reduce system resource consumption - - Add the capacity-related configuration parameter to limit the the data storage capacity + - Add the capacity-related configuration parameter to limit the data storage capacity + Tools @@ -160,7 +160,7 @@ TiDB version: 4.0.0-rc.2 - Fix the issue that the backup data cannot be restored from GCS [#7739](https://github.com/tikv/tikv/pull/7739) - Fix the issue that KMS key ID is not validated during encryption at rest [#7719](https://github.com/tikv/tikv/pull/7719) - Fix the underlying correctness issue of the Coprocessor in compilers of different architecture [#7714](https://github.com/tikv/tikv/pull/7714) [#7730](https://github.com/tikv/tikv/pull/7730) - - Fix the `snapshot ingestion` error when encrytion is enabled [#7815](https://github.com/tikv/tikv/pull/7815) + - Fix the `snapshot ingestion` error when encryption is enabled [#7815](https://github.com/tikv/tikv/pull/7815) - Fix the `Invalid cross-device link` error when rewriting the configuration file [#7817](https://github.com/tikv/tikv/pull/7817) - Fix the issue of wrong toml format when writing the configuration file to an empty file [#7817](https://github.com/tikv/tikv/pull/7817) - Fix the issue that a destroyed peer in Raftstore can still process requests [#7836](https://github.com/tikv/tikv/pull/7836) diff --git a/releases/release-4.0.0-rc.md b/releases/release-4.0.0-rc.md index 4a6b9b5ffed33..a2100bbfb387e 100644 --- a/releases/release-4.0.0-rc.md +++ b/releases/release-4.0.0-rc.md @@ -1,6 +1,6 @@ --- title: TiDB 4.0 RC Release Notes -aliases: ['/docs/dev/releases/release-4.0.0-rc/','/docs/dev/releases/4.0.0-rc/'] +summary: TiDB 4.0 RC released on April 8, 2020. It includes compatibility changes, bug fixes, new features, and tools. TiKV supports the `pipelined` feature in pessimistic transactions, improving TPC-C performance by 20%. TiDB adds case-insensitive collation and enhances the `RECOVER TABLE` syntax. TiKV now supports TLS in the HTTP port. PD supports getting default PD configuration information through the HTTP API. Bug fixes include issues with replication, subquery results, and DDL job internal retry. Tools like TiDB Lightning and TiCDC also have bug fixes and new features. --- # TiDB 4.0 RC Release Notes @@ -37,7 +37,7 @@ TiUP version: 0.0.3 + TiDB - Fix the issue that replication between the upstream and downstream might go wrong when the DDL job is executed using the `PREPARE` statement because of the incorrect job query in the internal records [#15435](https://github.com/pingcap/tidb/pull/15435) - - Fix the issue of incorrect subquery result in the `Read Commited` isolation level [#15471](https://github.com/pingcap/tidb/pull/15471) + - Fix the issue of incorrect subquery result in the `Read Committed` isolation level [#15471](https://github.com/pingcap/tidb/pull/15471) - Fix the issue of incorrect results caused by the Inline Projection optimization [#15411](https://github.com/pingcap/tidb/pull/15411) - Fix the issue that the SQL Hint `INL_MERGE_JOIN` is executed incorrectly in some cases [#15515](https://github.com/pingcap/tidb/pull/15515) - Fix the issue that columns with the `AutoRandom` attribute are rebased when the negative number is explicitly written to these columns [#15397](https://github.com/pingcap/tidb/pull/15397) @@ -48,7 +48,7 @@ TiUP version: 0.0.3 - Add the case-insensitive collation so that users can enable `utf8mb4_general_ci` and `utf8_general_ci` in a new cluster [#33](https://github.com/pingcap/tidb/projects/33) - Enhance the `RECOVER TABLE` syntax to support recovering truncated tables [#15398](https://github.com/pingcap/tidb/pull/15398) - - Refuse to get started instead of returning an alert log when the the tidb-server status port is occupied [#15177](https://github.com/pingcap/tidb/pull/15177) + - Refuse to get started instead of returning an alert log when the tidb-server status port is occupied [#15177](https://github.com/pingcap/tidb/pull/15177) - Optimize the write performance of using a sequence as the default column values [#15216](https://github.com/pingcap/tidb/pull/15216) - Add the `DDLJobs` system table to query the details of DDL jobs [#14837](https://github.com/pingcap/tidb/pull/14837) - Optimize the `aggFuncSum` performance [#14887](https://github.com/pingcap/tidb/pull/14887) @@ -79,7 +79,7 @@ TiUP version: 0.0.3 + TiDB - Fix the issue that replication between the upstream and downstream might go wrong when the DDL job is executed using the `PREPARE` statement because of the incorrect job query in the internal records [#15435](https://github.com/pingcap/tidb/pull/15435) - - Fix the issue of incorrect subquery result in the `Read Commited` isolation level [#15471](https://github.com/pingcap/tidb/pull/15471) + - Fix the issue of incorrect subquery result in the `Read Committed` isolation level [#15471](https://github.com/pingcap/tidb/pull/15471) - Fix the issue of possible wrong behavior when using `INSERT ... VALUES` to specify the `BIT(N)` data type [#15350](https://github.com/pingcap/tidb/pull/15350) - Fix the issue that the DDL Job internal retry does not fully achieve the expected outcomes because the values of `ErrorCount` fail to be summed correctly [#15373](https://github.com/pingcap/tidb/pull/15373) - Fix the issue that Garbage Collection might work abnormally when TiDB connects to TiFlash [#15505](https://github.com/pingcap/tidb/pull/15505) diff --git a/releases/release-4.0.1.md b/releases/release-4.0.1.md index 733e262e7bfe9..e2a6363c0d223 100644 --- a/releases/release-4.0.1.md +++ b/releases/release-4.0.1.md @@ -1,6 +1,6 @@ --- title: TiDB 4.0.1 Release Notes -aliases: ['/docs/dev/releases/release-4.0.1/'] +summary: TiDB 4.0.1 was released on June 12, 2020. New features include support for custom timeout for PD client and new collation framework in TiFlash. Bug fixes address issues with configuration, monitoring metrics, and store information retrieval. Backup & Restore (BR) now includes a version check to avoid compatibility issues. --- # TiDB 4.0.1 Release Notes diff --git a/releases/release-4.0.10.md b/releases/release-4.0.10.md index 2269f123bd87f..491cb43364d50 100644 --- a/releases/release-4.0.10.md +++ b/releases/release-4.0.10.md @@ -1,5 +1,6 @@ --- title: TiDB 4.0.10 Release Notes +summary: TiDB 4.0.10 was released on January 15, 2021. New features include redacting user data from logs and configurable size limits for key-value entries. Bug fixes address concurrency issues, duplicate bindings, and incorrect results. Improvements include optimized metrics and upgraded dependencies. Various tools have also been updated and fixed, such as TiCDC, Dumpling, Backup & Restore, TiDB Binlog, and TiDB Lightning. --- # TiDB 4.0.10 Release Notes @@ -33,7 +34,6 @@ TiDB version: 4.0.10 + TiCDC - - Enable the old value feature for the `maxwell` protocol [#1144](https://github.com/pingcap/tiflow/pull/1144) - Enable the unified sorter feature by default [#1230](https://github.com/pingcap/tiflow/pull/1230) + Dumpling @@ -82,7 +82,6 @@ TiDB version: 4.0.10 + TiCDC - - Fix the `maxwell` protocol issues, including the issue of `base64` data output and the issue of outputting TSO to unix timestamp [#1173](https://github.com/pingcap/tiflow/pull/1173) - Fix a bug that outdated metadata might cause the newly created changefeed abnormal [#1184](https://github.com/pingcap/tiflow/pull/1184) - Fix the issue of creating the receiver on the closed notifier [#1199](https://github.com/pingcap/tiflow/pull/1199) - Fix a bug that the TiCDC owner might consume too much memory in the etcd watch client [#1227](https://github.com/pingcap/tiflow/pull/1227) diff --git a/releases/release-4.0.11.md b/releases/release-4.0.11.md index dd5a339efc9f7..437bddc8f0b31 100644 --- a/releases/release-4.0.11.md +++ b/releases/release-4.0.11.md @@ -1,5 +1,6 @@ --- title: TiDB 4.0.11 Release Notes +summary: TiDB 4.0.11 was released on February 26, 2021. New features include support for `utf8_unicode_ci` and `utf8mb4_unicode_ci` collations. Improvements were made to inner joins, Grafana dashboards, and slow query metrics. Bug fixes address issues with collation, type inference, and function errors. TiKV improvements include support for multiple clusters in Grafana dashboards and bug fixes for memory diagnostics and OOM errors. PD fixes member health metrics and store limit persistence issues. TiFlash bug fixes address decimal type results, data loss, and crash issues. Tools like TiCDC, BR, and TiDB Lightning also received bug fixes and improvements. --- # TiDB 4.0.11 Release Notes diff --git a/releases/release-4.0.12.md b/releases/release-4.0.12.md index 4b61498987779..294ab24a2a876 100644 --- a/releases/release-4.0.12.md +++ b/releases/release-4.0.12.md @@ -1,5 +1,6 @@ --- title: TiDB 4.0.12 Release Notes +summary: TiDB 4.0.12 was released on April 2, 2021. New features include tools to check the status of `tiflash replica` for online rolling updates. Improvements were made to TiDB, TiKV, PD, TiFlash, and various tools. Bug fixes were also implemented for TiDB, TiKV, PD, TiFlash, TiCDC, Backup & Restore, and TiDB Lightning. --- # TiDB 4.0.12 Release Notes diff --git a/releases/release-4.0.13.md b/releases/release-4.0.13.md index 907e214d322c8..626a17dd82aa4 100644 --- a/releases/release-4.0.13.md +++ b/releases/release-4.0.13.md @@ -1,5 +1,6 @@ --- title: TiDB 4.0.13 Release Notes +summary: TiDB 4.0.13 was released on May 28, 2021. New features include support for changing `AUTO_INCREMENT` to `AUTO_RANDOM` and the addition of `infoschema.client_errors_summary` tables. Improvements were made to TiDB, TiKV, PD, TiFlash, and Tools. Bug fixes were also implemented for TiDB, TiKV, TiFlash, and Tools, addressing various issues such as query results, panics, and memory usage. --- # TiDB 4.0.13 Release Notes diff --git a/releases/release-4.0.14.md b/releases/release-4.0.14.md index 696cfbd82ba18..68c26ad92b324 100644 --- a/releases/release-4.0.14.md +++ b/releases/release-4.0.14.md @@ -1,5 +1,6 @@ --- title: TiDB 4.0.14 Release Notes +summary: TiDB 4.0.14 was released on July 27, 2021. The release includes compatibility changes, feature enhancements, improvements, bug fixes, and updates to various tools. Some notable changes include default value updates for TiDB and TiKV, support for OIDC SSO in TiDB Dashboard, and bug fixes for TiDB, TiKV, PD, TiFlash, and various tools. --- # TiDB 4.0.14 Release Notes diff --git a/releases/release-4.0.15.md b/releases/release-4.0.15.md index 4ea4e63661077..27074782d296b 100644 --- a/releases/release-4.0.15.md +++ b/releases/release-4.0.15.md @@ -1,5 +1,6 @@ --- title: TiDB 4.0.15 Release Notes +summary: "TiDB 4.0.15 Release Notes: Compatibility changes include bug fixes that might cause upgrade incompatibilities. Feature enhancements for TiKV support changing configurations dynamically. Improvements for TiDB, TiKV, PD, and Tools. Bug fixes for TiDB, TiKV, PD, TiFlash, Backup & Restore, and TiCDC." --- # TiDB 4.0.15 Release Notes diff --git a/releases/release-4.0.16.md b/releases/release-4.0.16.md index 429d8a7bf8db3..c552efed45683 100644 --- a/releases/release-4.0.16.md +++ b/releases/release-4.0.16.md @@ -1,5 +1,6 @@ --- title: TiDB 4.0.16 Release Notes +summary: TiDB 4.0.16 was released on December 17, 2021. The release includes compatibility changes for TiKV and Tools, improvements for TiDB, TiKV, and Tools, bug fixes for TiDB, TiKV, PD, TiFlash, TiDB Binlog, and TiCDC. The bug fixes address various issues such as query panics, wrong results, panics, and memory leaks. The release also includes fixes for TiCDC replication interruption, OOM in container environments, and memory leak issues. --- # TiDB 4.0.16 Release Notes @@ -80,7 +81,7 @@ TiDB version: 4.0.16 + PD - Fix a panic issue that occurs after the TiKV node is removed [#4344](https://github.com/tikv/pd/issues/4344) - - Fix slow leader election caused by stucked region syncer [#3936](https://github.com/tikv/pd/issues/3936) + - Fix slow leader election caused by stuck region syncer [#3936](https://github.com/tikv/pd/issues/3936) - Support that the evict leader scheduler can schedule regions with unhealthy peers [#4093](https://github.com/tikv/pd/issues/4093) + TiFlash diff --git a/releases/release-4.0.2.md b/releases/release-4.0.2.md index 73f3990b71dd5..51fa62791efb3 100644 --- a/releases/release-4.0.2.md +++ b/releases/release-4.0.2.md @@ -1,6 +1,6 @@ --- title: TiDB 4.0.2 Release Notes -aliases: ['/docs/dev/releases/release-4.0.2/'] +summary: TiDB 4.0.2 was released on July 1, 2020. The new version includes compatibility changes, new features, improvements, bug fixes, and new changes. Some highlights include support for new aggregate functions, improvements in query latency, and bug fixes related to execution plan, runtime errors, and data replication. Additionally, there are new features and improvements in TiKV, PD, TiFlash, and Tools. --- # TiDB 4.0.2 Release Notes diff --git a/releases/release-4.0.3.md b/releases/release-4.0.3.md index 5a8ce13d151d7..81ab47b39a366 100644 --- a/releases/release-4.0.3.md +++ b/releases/release-4.0.3.md @@ -1,5 +1,6 @@ --- title: TiDB 4.0.3 Release Notes +summary: TiDB 4.0.3 was released on July 24, 2020. New features include TiDB Dashboard improvements, TiFlash file encryption, and support for various tools. Improvements were made to TiDB, TiKV, PD, and TiDB Dashboard. Bug fixes were also implemented for TiDB, TiKV, PD, TiDB Dashboard, TiFlash, TiCDC, Backup & Restore, Dumpling, TiDB Lightning, and TiDB Binlog. --- # TiDB 4.0.3 Release Notes diff --git a/releases/release-4.0.4.md b/releases/release-4.0.4.md index 2d5d9b0d58204..837a9796b2445 100644 --- a/releases/release-4.0.4.md +++ b/releases/release-4.0.4.md @@ -1,5 +1,6 @@ --- title: TiDB 4.0.4 Release Notes +summary: TiDB 4.0.4 was released on July 31, 2020. Bug fixes include issues with querying `information_schema.columns`, errors with `PointGet` and `BatchPointGet` operators, wrong results with `BatchPointGet`, and incorrect query results with the `HashJoin` operator encountering `set` or `enum` type. --- # TiDB 4.0.4 Release Notes diff --git a/releases/release-4.0.5.md b/releases/release-4.0.5.md index fff905897f804..8fda3f47a9514 100644 --- a/releases/release-4.0.5.md +++ b/releases/release-4.0.5.md @@ -1,5 +1,6 @@ --- title: TiDB 4.0.5 Release Notes +summary: TiDB 4.0.5 was released on August 31, 2020. The new version includes compatibility changes, new features, improvements, bug fixes, and updates to TiKV, TiFlash, Tools, PD, and TiDB Lightning. Some notable changes include support for the unified log format with TiDB, optimization of performance, bug fixes for various issues, and support for encryption at rest for data storage in TiFlash. --- # TiDB 4.0.5 Release Notes diff --git a/releases/release-4.0.6.md b/releases/release-4.0.6.md index 82d4aac631322..c15f5f6a050db 100644 --- a/releases/release-4.0.6.md +++ b/releases/release-4.0.6.md @@ -1,5 +1,6 @@ --- title: TiDB 4.0.6 Release Notes +summary: TiDB 4.0.6 was released on September 15, 2020. New features include TiFlash support for outer join and TiDB Dashboard improvements. Tools like TiCDC and TiKV have also been updated. Bug fixes for TiDB, TiKV, PD, TiFlash, and various tools are included in this release. --- # TiDB 4.0.6 Release Notes @@ -26,8 +27,6 @@ TiDB version: 4.0.6 + TiCDC (GA since v4.0.6) - - Support outputting data in the `maxwell` format [#869](https://github.com/pingcap/tiflow/pull/869) - ## Improvements + TiDB @@ -105,7 +104,7 @@ TiDB version: 4.0.6 - Fix a bug of converting the `enum` and `set` types [#19778](https://github.com/pingcap/tidb/pull/19778) - Add a privilege check for `SHOW STATS_META` and `SHOW STATS_BUCKET` [#19760](https://github.com/pingcap/tidb/pull/19760) - Fix the error of unmatched column lengths caused by `builtinGreatestStringSig` and `builtinLeastStringSig` [#19758](https://github.com/pingcap/tidb/pull/19758) - - If unnecessary errors or warnings occur, the vectorized control expresions fall back to their scalar execution [#19749](https://github.com/pingcap/tidb/pull/19749) + - If unnecessary errors or warnings occur, the vectorized control expression fall back to their scalar execution [#19749](https://github.com/pingcap/tidb/pull/19749) - Fix the error of the `Apply` operator when the type of the correlation column is `Bit` [#19692](https://github.com/pingcap/tidb/pull/19692) - Fix the issue that occurs when the user queries `processlist` and `cluster_log` in MySQL 8.0 client [#19690](https://github.com/pingcap/tidb/pull/19690) - Fix the issue that plans of the same type have different plan digests [#19684](https://github.com/pingcap/tidb/pull/19684) diff --git a/releases/release-4.0.7.md b/releases/release-4.0.7.md index 972469561106e..b0b220cfb5a0f 100644 --- a/releases/release-4.0.7.md +++ b/releases/release-4.0.7.md @@ -1,5 +1,6 @@ --- title: TiDB 4.0.7 Release Notes +summary: TiDB 4.0.7 was released on September 29, 2020. New features include the addition of the `GetAllMembers` function in the PD client and support for generating the metrics relationship graph in TiDB Dashboard. Improvements were made to TiDB, TiKV, PD, TiFlash, and various tools. Bug fixes were also implemented for TiDB, TiKV, PD, TiFlash, and tools like Backup & Restore and Dumpling. --- # TiDB 4.0.7 Release Notes diff --git a/releases/release-4.0.8.md b/releases/release-4.0.8.md index efa6c6d8ad0ee..7d90e89ae0165 100644 --- a/releases/release-4.0.8.md +++ b/releases/release-4.0.8.md @@ -1,5 +1,6 @@ --- title: TiDB 4.0.8 Release Notes +summary: TiDB 4.0.8 was released on October 30, 2020. New features include support for the new aggregate function `APPROX_PERCENTILE` and pushing down `CAST` functions in TiFlash. Improvements were made to TiDB, TiKV, PD, and TiFlash. Bug fixes were also implemented for TiDB, TiKV, PD, TiFlash, Backup and Restore (BR), TiCDC, and TiDB Lightning. --- # TiDB 4.0.8 Release Notes @@ -148,7 +149,6 @@ TiDB version: 4.0.8 - Fix the unexpected exit caused by the failure to update the GC safepoint [#979](https://github.com/pingcap/tiflow/pull/979) - Fix the issue that the task status is unexpectedly flushed because of the incorrect mod revision cache [#1017](https://github.com/pingcap/tiflow/pull/1017) - - Fix the unexpected empty Maxwell messages [#978](https://github.com/pingcap/tiflow/pull/978) + TiDB Lightning diff --git a/releases/release-4.0.9.md b/releases/release-4.0.9.md index 016dede57f686..9b62baa1d96a9 100644 --- a/releases/release-4.0.9.md +++ b/releases/release-4.0.9.md @@ -1,5 +1,6 @@ --- title: TiDB 4.0.9 Release Notes +summary: TiDB 4.0.9 was released on December 21, 2020. The release includes compatibility changes, new features, improvements, bug fixes, and updates to TiKV, TiDB Dashboard, PD, TiFlash, and various tools. Notable changes include the deprecation of the `enable-streaming` configuration item in TiDB, support for storing the latest data of the storage engine on multiple disks in TiFlash, and various bug fixes in TiDB and TiKV. --- # TiDB 4.0.9 Release Notes diff --git a/releases/release-5.0.0-rc.md b/releases/release-5.0.0-rc.md index befdfb08b4236..f8a7f8ff444a9 100644 --- a/releases/release-5.0.0-rc.md +++ b/releases/release-5.0.0-rc.md @@ -1,5 +1,6 @@ --- title: TiDB 5.0 RC Release Notes +summary: TiDB v5.0.0-rc is the predecessor version of TiDB v5.0. It includes new features like clustered index, async commit, reduced jitters, Raft Joint Consensus algorithm, optimized `EXPLAIN` features, invisible index, and improved reliability for enterprise data. It also supports desensitizing error messages and log files for security. Performance improvements include async commit, optimizer stability, and reduced performance jitter. It also enhances system availability during Region membership change. Additionally, it supports backup and restore to AWS S3 and Google Cloud GCS, data import/export, and optimized `EXPLAIN` features for troubleshooting SQL performance issues. Deployment and maintenance improvements include enhanced `mirror` command and easier installation process. --- # TiDB 5.0 RC Release Notes diff --git a/releases/release-5.0.0.md b/releases/release-5.0.0.md index 3075ec4c22123..7bc62299d4fbc 100644 --- a/releases/release-5.0.0.md +++ b/releases/release-5.0.0.md @@ -1,5 +1,6 @@ --- title: What's New in TiDB 5.0 +summary: TiDB 5.0 introduces MPP architecture, clustered index, async commit, and stability improvements. It also enhances compatibility changes, configuration parameters, and new features. Additionally, it optimizes performance, high availability, disaster recovery, data migration, diagnostics, deployment, and maintenance. Telemetry is added for cluster usage metrics. --- # What's New in TiDB 5.0 @@ -176,7 +177,7 @@ Currently, the main features that the MPP mode does not support are as follows ( When you are designing table structures or analyzing database behaviors, it is recommended to use the clustered index feature if you find that some columns with primary keys are often grouped and sorted, queries on these columns often return a certain range of data or a small amount of data with different values, and the corresponding data does not cause read or write hotspot issues. -Clustered indexes, also known as _index-organized tables_ in some database management systems, is a storage structure associated with the data of a table. When creating a clustered index, you can specify one or more columns from the table as the keys for the index. TiDB stores these keys in a specific structure, which allows TiDB to quickly and efficiently find the rows associated with the keys, thus improves the performance of querying and writing data. +Clustered indexes is a storage structure associated with the data of a table. Some database management systems refer to clustered index tables as _index-organized tables_. When creating a clustered index, you can specify one or more columns from the table as the keys for the index. TiDB stores these keys in a specific structure, which allows TiDB to quickly and efficiently find the rows associated with the keys, thus improves the performance of querying and writing data. When the clustered index feature is enabled, the TiDB performance improves significantly (for example in the TPC-C tpmC test, the performance of TiDB, with clustered index enabled, improves by 39%) in the following cases: @@ -333,7 +334,7 @@ You can view the manually or automatically bound execution plan information by r When upgrading TiDB, to avoid performance jitter, you can enable the baseline capturing feature to allow the system to automatically capture and bind the latest execution plan and store it in the system table. After TiDB is upgraded, you can export the bound execution plan by running the `SHOW GLOBAL BINDING` command and decide whether to delete these plans. -This feature is disbled by default. You can enable it by modifying the server or setting the `tidb_capture_plan_baselines` global system variable to `ON`. When this feature is enabled, the system fetches the SQL statements that appear at least twice from the Statement Summary every `bind-info-lease` (the default value is `3s`), and automatically captures and binds these SQL statements. +This feature is disabled by default. You can enable it by modifying the server or setting the `tidb_capture_plan_baselines` global system variable to `ON`. When this feature is enabled, the system fetches the SQL statements that appear at least twice from the Statement Summary every `bind-info-lease` (the default value is `3s`), and automatically captures and binds these SQL statements. ### Improve stability of TiFlash queries diff --git a/releases/release-5.0.1.md b/releases/release-5.0.1.md index 69b1dba224d93..232311b15d0e1 100644 --- a/releases/release-5.0.1.md +++ b/releases/release-5.0.1.md @@ -1,5 +1,6 @@ --- title: TiDB 5.0.1 Release Notes +summary: TiDB 5.0.1 was released on April 24, 2021. The default value of `committer-concurrency` changed to 128. Various bug fixes and improvements were made to TiDB, TiKV, PD, TiFlash, and Tools. For example, TiDB fixed issues with query results and performance regression, while TiKV fixed issues with coprocessors and startup failures. Tools like TiDB Lightning and Backup & Restore also received bug fixes. --- # TiDB 5.0.1 Release Notes diff --git a/releases/release-5.0.2.md b/releases/release-5.0.2.md index 51086ad5ddfad..d4d5b30607151 100644 --- a/releases/release-5.0.2.md +++ b/releases/release-5.0.2.md @@ -1,5 +1,6 @@ --- title: TiDB 5.0.2 Release Notes +summary: TiDB 5.0.2 was released on June 10, 2021. The new version includes compatibility changes, new features, improvements, bug fixes, and updates to various tools such as TiKV, TiFlash, PD, TiCDC, Backup & Restore (BR), and TiDB Lightning. Some notable changes include the deprecation of `--sort-dir` in TiCDC, enabling the Hibernate Region feature in TiKV, and various bug fixes in TiDB, TiKV, PD, TiFlash, and tools like TiCDC, BR, and TiDB Lightning. --- # TiDB 5.0.2 Release Notes diff --git a/releases/release-5.0.3.md b/releases/release-5.0.3.md index eb29090c88965..038bd3dfc8bc4 100644 --- a/releases/release-5.0.3.md +++ b/releases/release-5.0.3.md @@ -1,5 +1,6 @@ --- title: TiDB 5.0.3 Release Notes +summary: TiDB 5.0.3 was released on July 2, 2021. The release includes compatibility changes, feature enhancements, improvements, bug fixes, and updates for TiDB, TiKV, PD, TiFlash, and Tools like TiCDC, Backup & Restore (BR), and TiDB Lightning. Some notable changes include support for pushing down operators and functions to TiFlash, memory consumption limits for TiCDC, and bug fixes for various issues in TiDB, TiKV, PD, and TiFlash. --- # TiDB 5.0.3 Release Notes diff --git a/releases/release-5.0.4.md b/releases/release-5.0.4.md index fc7ad9805dcb9..be37425e8d9c6 100644 --- a/releases/release-5.0.4.md +++ b/releases/release-5.0.4.md @@ -1,5 +1,6 @@ --- title: TiDB 5.0.4 Release Notes +summary: Compatibility changes include fixes for slow `SHOW VARIABLES` execution, default value change for `tidb_stmt_summary_max_stmt_count`, and bug fixes that may cause upgrade incompatibilities. Feature enhancements include support for setting `tidb_enforce_mpp=1` and dynamic TiCDC configurations. Improvements cover auto-analyze trigger, MPP query retry support, and stable result mode. Bug fixes address various issues in TiDB, TiKV, PD, TiFlash, and tools like Dumpling and TiCDC. --- # TiDB 5.0.4 Release Notes diff --git a/releases/release-5.0.5.md b/releases/release-5.0.5.md index ebc8bc4705501..341eb2e955574 100644 --- a/releases/release-5.0.5.md +++ b/releases/release-5.0.5.md @@ -1,5 +1,6 @@ --- title: TiDB 5.0.5 Release Note +summary: TiDB 5.0.5 was released on December 3, 2021. The bug fix for TiKV addresses an issue where the `GcKeys` task does not work when called by multiple keys, causing compaction filter GC to not drop MVCC deletion information. Issue #11217 on GitHub provides more details. --- # TiDB 5.0.5 Release Note diff --git a/releases/release-5.0.6.md b/releases/release-5.0.6.md index 03c185b560291..a6a1454bbaba7 100644 --- a/releases/release-5.0.6.md +++ b/releases/release-5.0.6.md @@ -1,6 +1,7 @@ --- title: TiDB 5.0.6 Release Notes category: Releases +summary: TiDB 5.0.6 was released on December 31, 2021. The release includes compatibility changes, improvements, bug fixes, and updates to various tools such as TiCDC, TiKV, PD, TiDB Lightning, TiFlash, Backup & Restore (BR), and Dumpling. The changes include enhancements to error handling, performance improvements, bug fixes related to SQL statements, and various optimizations for different tools. --- # TiDB 5.0.6 Release Notes @@ -76,7 +77,7 @@ TiDB version: 5.0.6 - Fix the `INDEX OUT OF RANGE` error for a MPP query after deleting an empty `dual table` [#28250](https://github.com/pingcap/tidb/issues/28250) - Fix the TiDB panic when inserting invalid date values concurrently [#25393](https://github.com/pingcap/tidb/issues/25393) - Fix the unexpected `can not found column in Schema column` error for queries in the MPP mode [#30980](https://github.com/pingcap/tidb/issues/30980) - - Fix the issue that TiDB might panic when TiFlash is shuting down [#28096](https://github.com/pingcap/tidb/issues/28096) + - Fix the issue that TiDB might panic when TiFlash is shutting down [#28096](https://github.com/pingcap/tidb/issues/28096) - Fix the unexpected `index out of range` error when the planner is doing join reorder [#24095](https://github.com/pingcap/tidb/issues/24095) - Fix wrong results of the control functions (such as `IF` and `CASE WHEN`) when using the `ENUM` type data as parameters of such functions [#23114](https://github.com/pingcap/tidb/issues/23114) - Fix the wrong result of `CONCAT(IFNULL(TIME(3))` [#29498](https://github.com/pingcap/tidb/issues/29498) @@ -114,7 +115,7 @@ TiDB version: 5.0.6 - Fix a panic issue that occurs after the TiKV node is removed [#4344](https://github.com/tikv/pd/issues/4344) - Fix the issue that operator can get blocked due to down store [#3353](https://github.com/tikv/pd/issues/3353) - - Fix slow leader election caused by stucked Region syncer [#3936](https://github.com/tikv/pd/issues/3936) + - Fix slow leader election caused by stuck Region syncer [#3936](https://github.com/tikv/pd/issues/3936) - Fix the issue that the speed of removing peers is limited when repairing the down nodes [#4090](https://github.com/tikv/pd/issues/4090) - Fix the issue that the hotspot cache cannot be cleared when the Region heartbeat is less than 60 seconds [#4390](https://github.com/tikv/pd/issues/4390) @@ -150,7 +151,7 @@ TiDB version: 5.0.6 - Fix the issue that Avro sink does not support parsing JSON type columns [#3624](https://github.com/pingcap/tiflow/issues/3624) - Fix the bug that TiCDC reads the incorrect schema snapshot from TiKV when the TiKV owner restarts [#2603](https://github.com/pingcap/tiflow/issues/2603) - Fix the memory leak issue after processing DDLs [#3174](https://github.com/pingcap/ticdc/issues/3174) - - Fix the bug that the `enable-old-value` configuration item is not automatically set to `true` on Canal and Maxwell protocols [#3676](https://github.com/pingcap/tiflow/issues/3676) + - Fix the bug that the `enable-old-value` configuration item is not automatically set to `true` on the Canal protocol [#3676](https://github.com/pingcap/tiflow/issues/3676) - Fix the timezone error that occurs when the `cdc server` command runs on some Red Hat Enterprise Linux releases (such as 6.8 and 6.9) [#3584](https://github.com/pingcap/tiflow/issues/3584) - Fix the issue of the inaccurate `txn_batch_size` monitoring metric for Kafka sink [#3431](https://github.com/pingcap/tiflow/issues/3431) - Fix the issue that `tikv_cdc_min_resolved_ts_no_change_for_1m` keeps alerting when there is no changefeed [#11017](https://github.com/tikv/tikv/issues/11017) diff --git a/releases/release-5.1.0.md b/releases/release-5.1.0.md index 704cb272766f6..608bad96342f1 100644 --- a/releases/release-5.1.0.md +++ b/releases/release-5.1.0.md @@ -1,5 +1,6 @@ --- title: TiDB 5.1 Release Notes +summary: TiDB 5.1 introduces support for Common Table Expression, dynamic privilege feature, and Stale Read. It also includes new statistics type, Lock View feature, and TiKV write rate limiter. Compatibility changes include new system and configuration variables. Other improvements and bug fixes are also part of this release. --- # TiDB 5.1 Release Notes diff --git a/releases/release-5.1.1.md b/releases/release-5.1.1.md index 1794ec951e1c4..80fa9312e170c 100644 --- a/releases/release-5.1.1.md +++ b/releases/release-5.1.1.md @@ -1,5 +1,6 @@ --- title: TiDB 5.1.1 Release Notes +summary: TiDB 5.1.1 was released on July 30, 2021. The release includes compatibility changes, feature enhancements, improvements, bug fixes, and updates to TiDB Dashboard, TiFlash, TiKV, and various tools. Notable changes include default value changes for variables, support for OIDC SSO in TiDB Dashboard, and bug fixes for data loss and panic issues. --- # TiDB 5.1.1 Release Notes diff --git a/releases/release-5.1.2.md b/releases/release-5.1.2.md index 5ab837992eb03..47700e11fabba 100644 --- a/releases/release-5.1.2.md +++ b/releases/release-5.1.2.md @@ -1,5 +1,6 @@ --- title: TiDB 5.1.2 Release Notes +summary: TiDB 5.1.2 was released on September 27, 2021. The release includes compatibility changes, improvements, bug fixes, and updates to various tools such as TiCDC, TiKV, PD, TiFlash, BR, Dumpling, and TiCDC. The release addresses numerous bug fixes and improvements to enhance performance and stability. --- # TiDB 5.1.2 Release Notes diff --git a/releases/release-5.1.3.md b/releases/release-5.1.3.md index f90868b6e94b4..1f32285f9ab7b 100644 --- a/releases/release-5.1.3.md +++ b/releases/release-5.1.3.md @@ -1,5 +1,6 @@ --- title: TiDB 5.1.3 Release Note +summary: TiDB 5.1.3 was released on December 3, 2021. This version includes a bug fix for TiKV, addressing an issue where the `GcKeys` task does not work when called by multiple keys, leading to potential problems with compaction filter GC. --- # TiDB 5.1.3 Release Note diff --git a/releases/release-5.1.4.md b/releases/release-5.1.4.md index 679389c649a7b..aea49c76ca4e9 100644 --- a/releases/release-5.1.4.md +++ b/releases/release-5.1.4.md @@ -1,5 +1,6 @@ --- title: TiDB 5.1.4 Release Notes +summary: "TiDB 5.1.4 Release Notes: Compatibility changes include default value changes for system variables. Improvements in partition pruning, memory usage tracking, and speed of inserting SST files. Bug fixes address memory leaks, configuration issues, and incorrect query results. Tools like TiCDC and TiFlash also receive various fixes and improvements." --- # TiDB 5.1.4 Release Notes @@ -109,7 +110,7 @@ TiDB version: 5.1.4 - Fix a bug that the schedule generated by the region scatterer might decrease the number of peers [#4565](https://github.com/tikv/pd/issues/4565) - Fix the issue that Region statistics are not affected by `flow-round-by-digit` [#4295](https://github.com/tikv/pd/issues/4295) - - Fix slow leader election caused by stucked region syncer [#3936](https://github.com/tikv/pd/issues/3936) + - Fix slow leader election caused by stuck region syncer [#3936](https://github.com/tikv/pd/issues/3936) - Support that the evict leader scheduler can schedule regions with unhealthy peers [#4093](https://github.com/tikv/pd/issues/4093) - Fix the issue that the cold hotspot data cannot be deleted from the hotspot statistics [#4390](https://github.com/tikv/pd/issues/4390) - Fix a panic issue that occurs after the TiKV node is removed [#4344](https://github.com/tikv/pd/issues/4344) @@ -152,7 +153,7 @@ TiDB version: 5.1.4 - Fix the TiCDC panic issue that occurs when manually cleaning the task status in etcd [#2980](https://github.com/pingcap/tiflow/issues/2980) - Fix the issue that the service cannot be started because of a timezone issue in the RHEL release [#3584](https://github.com/pingcap/tiflow/issues/3584) - Fix the issue of overly frequent warnings caused by MySQL sink deadlock [#2706](https://github.com/pingcap/tiflow/issues/2706) - - Fix the bug that the `enable-old-value` configuration item is not automatically set to `true` on Canal and Maxwell protocols [#3676](https://github.com/pingcap/tiflow/issues/3676) + - Fix the bug that the `enable-old-value` configuration item is not automatically set to `true` on the Canal protocol [#3676](https://github.com/pingcap/tiflow/issues/3676) - Fix the issue that Avro sink does not support parsing JSON type columns [#3624](https://github.com/pingcap/tiflow/issues/3624) - Fix the negative value error in the changefeed checkpoint lag [#3010](https://github.com/pingcap/ticdc/issues/3010) - Fix the OOM issue in the container environment [#1798](https://github.com/pingcap/tiflow/issues/1798) diff --git a/releases/release-5.1.5.md b/releases/release-5.1.5.md index 213829664ee6c..43c9837f220b7 100644 --- a/releases/release-5.1.5.md +++ b/releases/release-5.1.5.md @@ -1,5 +1,6 @@ --- title: TiDB 5.1.5 Release Notes +summary: TiDB 5.1.5 was released on December 28, 2022. The release includes compatibility changes and numerous bug fixes for TiDB, TiKV, PD, TiFlash, and various tools. Bug fixes address issues such as panics, wrong results, and incorrect behaviors. Fixes also include issues related to data loss, memory usage, and incorrect metrics. --- # TiDB 5.1.5 Release Notes @@ -8,7 +9,7 @@ Release date: December 28, 2022 TiDB version: 5.1.5 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v5.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v5.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v5.1.5#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v5.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v5.1/production-deployment-using-tiup) ## Compatibility changes diff --git a/releases/release-5.2.0.md b/releases/release-5.2.0.md index ed6f6e0ebe14f..40f4df5a0374a 100644 --- a/releases/release-5.2.0.md +++ b/releases/release-5.2.0.md @@ -1,5 +1,6 @@ --- title: TiDB 5.2 Release Notes +summary: TiDB 5.2.0 introduces new features and improvements, including support for expression indexes, Lock View GA, TiFlash I/O traffic limit, and more. Compatibility changes include new system variables and configuration file parameters. The release also includes bug fixes and feature enhancements for TiDB, TiKV, TiFlash, and tools like TiCDC, BR, Lightning, and Dumpling. --- # TiDB 5.2 Release Notes @@ -210,7 +211,7 @@ Support running the `tiup playground` command on Mac computers with Apple M1 chi - Support completing the garbage collection automatically for the bindings in the "deleted" status [#26206](https://github.com/pingcap/tidb/pull/26206) - Support showing whether a binding is used for query optimization in the result of `EXPLAIN VERBOSE` [#26930](https://github.com/pingcap/tidb/pull/26930) - Add a new status variation `last_plan_binding_update_time` to view the timestamp corresponding to the binding cache in the current TiDB instance [#26340](https://github.com/pingcap/tidb/pull/26340) - - Support reporting an error when starting binding evolution or running `admin evolve bindings` to ban the baseline evolution (currently disabled in the TiDB Self-Hosted version because it is an experimental feature) affecting other features [#26333](https://github.com/pingcap/tidb/pull/26333) + - Support reporting an error when starting binding evolution or running `admin evolve bindings` to ban the baseline evolution (currently disabled in the TiDB Self-Managed version because it is an experimental feature) affecting other features [#26333](https://github.com/pingcap/tidb/pull/26333) + PD diff --git a/releases/release-5.2.1.md b/releases/release-5.2.1.md index 55e7ee9b1c904..a344f6e5011c5 100644 --- a/releases/release-5.2.1.md +++ b/releases/release-5.2.1.md @@ -1,5 +1,6 @@ --- title: TiDB 5.2.1 Release Notes +summary: TiDB 5.2.1 was released on September 9, 2021. Bug fixes include resolving an error in TiDB caused by a wrong execution plan and fixing the issue of unavailable TiKV caused by Raftstore deadlock when migrating Regions. --- # TiDB 5.2.1 Release Notes diff --git a/releases/release-5.2.2.md b/releases/release-5.2.2.md index 9e2e748b99d34..7ba8c354199b9 100644 --- a/releases/release-5.2.2.md +++ b/releases/release-5.2.2.md @@ -1,5 +1,6 @@ --- title: TiDB 5.2.2 Release Notes +summary: TiDB 5.2.2 was released on October 29, 2021. The release includes various improvements and bug fixes for TiDB, TiKV, PD, TiCDC, TiFlash, and TiDB Binlog. Improvements include showing affected SQL statements in debug logs, support for showing backup and restore data size, and more. Bug fixes address issues such as plan-cache detection, wrong partition pruning, and various other issues related to query functions, client connections, and data replication. --- # TiDB 5.2.2 Release Notes @@ -62,7 +63,7 @@ TiDB version: 5.2.2 - Fix the `INDEX OUT OF RANGE` error for a MPP query after deleting an empty `dual table`. [#28250](https://github.com/pingcap/tidb/issues/28250) - Fix the issue of false positive error log `invalid cop task execution summaries length` for MPP queries [#1791](https://github.com/pingcap/tics/issues/1791) - Fix the issue of error log `cannot found column in Schema column` for MPP queries [#28149](https://github.com/pingcap/tidb/pull/28149) - - Fix the issue that TiDB might panic when TiFlash is shuting down [#28096](https://github.com/pingcap/tidb/issues/28096) + - Fix the issue that TiDB might panic when TiFlash is shutting down [#28096](https://github.com/pingcap/tidb/issues/28096) - Remove the support for insecure 3DES (Triple Data Encryption Algorithm) based TLS cipher suites [#27859](https://github.com/pingcap/tidb/pull/27859) - Fix the issue that Lightning connects to offline TiKV nodes during pre-check and causes import failures [#27826](https://github.com/pingcap/tidb/pull/27826) - Fix the issue that pre-check cost too much time when importing many files to tables [#27605](https://github.com/pingcap/tidb/issues/27605) diff --git a/releases/release-5.2.3.md b/releases/release-5.2.3.md index 039541f40536d..6b32b04a92384 100644 --- a/releases/release-5.2.3.md +++ b/releases/release-5.2.3.md @@ -1,5 +1,6 @@ --- title: TiDB 5.2.3 Release Note +summary: TiDB 5.2.3 was released on December 3, 2021. This version includes a bug fix for TiKV, addressing an issue where the `GcKeys` task does not work when called by multiple keys, leading to potential problems with compaction filter GC. (#11217) --- # TiDB 5.2.3 Release Note diff --git a/releases/release-5.2.4.md b/releases/release-5.2.4.md index 039dd7944d8cb..95b87a499a889 100644 --- a/releases/release-5.2.4.md +++ b/releases/release-5.2.4.md @@ -1,6 +1,7 @@ --- title: TiDB 5.2.4 Release Notes category: Releases +summary: Learn about the new features, compatibility changes, improvements, and bug fixes in TiDB 5.2.4. --- # TiDB 5.2.4 Release Notes @@ -88,7 +89,7 @@ TiDB version: 5.2.4 - Fix the issue that the system variable `max_allowed_packet` does not take effect [#31422](https://github.com/pingcap/tidb/issues/31422) - Fix the issue that the `REPLACE` statement incorrectly changes other rows when the auto ID is out of range [#29483](https://github.com/pingcap/tidb/issues/29483) - Fix the issue that the slow query log cannot output log normally and might consume too much memory [#32656](https://github.com/pingcap/tidb/issues/32656) - - Fix the issue that the result of NATURAL JOIN might include unexpected columns [#24981](https://github.com/pingcap/tidb/issues/29481) + - Fix the issue that the result of NATURAL JOIN might include unexpected columns [#29481](https://github.com/pingcap/tidb/issues/29481) - Fix the issue that using `ORDER BY` and `LIMIT` together in one statement might output wrong results if a prefix-column index is used to query data [#29711](https://github.com/pingcap/tidb/issues/29711) - Fix the issue that the DOUBLE type auto-increment column might be changed when the optimistic transaction retries [#29892](https://github.com/pingcap/tidb/issues/29892) - Fix the issue that the STR_TO_DATE function cannot handle the preceding zero of the microsecond part correctly [#30078](https://github.com/pingcap/tidb/issues/30078) @@ -165,7 +166,7 @@ TiDB version: 5.2.4 + TiCDC - Fix the issue that default values cannot be replicated [#3793](https://github.com/pingcap/tiflow/issues/3793) - - Fix a bug that sequence is incorrectly replicated in some cases [#4563](https://github.com/pingcap/tiflow/issues/4552) + - Fix a bug that sequence is incorrectly replicated in some cases [#4552](https://github.com/pingcap/tiflow/issues/4552) - Fix a bug that a TiCDC node exits abnormally when a PD leader is killed [#4248](https://github.com/pingcap/tiflow/issues/4248) - Fix a bug that MySQL sink generates duplicated `replace` SQL statements when `batch-replace-enable` is disabled [#4501](https://github.com/pingcap/tiflow/issues/4501) - Fix the issue of panic and data inconsistency that occurs when outputting the default column value [#3929](https://github.com/pingcap/tiflow/issues/3929) @@ -181,7 +182,7 @@ TiDB version: 5.2.4 - Fix the issue that the service cannot be started because of a timezone issue in the RHEL release [#3584](https://github.com/pingcap/tiflow/issues/3584) - Fix the issue that `stopped` changefeeds resume automatically after a cluster upgrade [#3473](https://github.com/pingcap/tiflow/issues/3473) - Fix the issue of overly frequent warnings caused by MySQL sink deadlock [#2706](https://github.com/pingcap/tiflow/issues/2706) - - Fix the bug that the `enable-old-value` configuration item is not automatically set to `true` on Canal and Maxwell protocols [#3676](https://github.com/pingcap/tiflow/issues/3676) + - Fix the bug that the `enable-old-value` configuration item is not automatically set to `true` on the Canal protocol [#3676](https://github.com/pingcap/tiflow/issues/3676) - Fix the issue that Avro sink does not support parsing JSON type columns [#3624](https://github.com/pingcap/tiflow/issues/3624) - Fix the negative value error in the changefeed checkpoint lag [#3010](https://github.com/pingcap/tiflow/issues/3010) - Fix the OOM issue in the container environment [#1798](https://github.com/pingcap/tiflow/issues/1798) diff --git a/releases/release-5.3.0.md b/releases/release-5.3.0.md index a7c026b88a0b1..0284ae17d7cf9 100644 --- a/releases/release-5.3.0.md +++ b/releases/release-5.3.0.md @@ -1,5 +1,6 @@ --- title: TiDB 5.3 Release Notes +summary: TiDB 5.3.0 introduces temporary tables, table attributes, and user privileges on TiDB Dashboard for improved performance and security. It also enhances TiDB Data Migration, supports parallel import using multiple TiDB Lightning instances, and continuous profiling for better observability. Compatibility changes and configuration file parameters have been modified. The release also includes new SQL features, security enhancements, stability improvements, and diagnostic efficiency. Additionally, bug fixes and improvements have been made to TiDB, TiKV, PD, TiFlash, and TiCDC. The cyclic replication feature between TiDB clusters has been removed. Telemetry now includes information about the usage of the TEMPORARY TABLE feature. --- # TiDB 5.3 Release Notes @@ -337,7 +338,7 @@ Starting from TiCDC v5.3.0, the cyclic replication feature between TiDB clusters - Fix the `INDEX OUT OF RANGE` error for a MPP query after deleting an empty `dual table` [#28250](https://github.com/pingcap/tidb/issues/28250) - Fix the issue of false positive error log `invalid cop task execution summaries length` for MPP queries [#1791](https://github.com/pingcap/tics/issues/1791) - Fix the issue of error log `cannot found column in Schema column` for MPP queries [#28149](https://github.com/pingcap/tidb/pull/28149) - - Fix the issue that TiDB might panic when TiFlash is shuting down [#28096](https://github.com/pingcap/tidb/issues/28096) + - Fix the issue that TiDB might panic when TiFlash is shutting down [#28096](https://github.com/pingcap/tidb/issues/28096) - Remove the support for insecure 3DES (Triple Data Encryption Algorithm) based TLS cipher suites [#27859](https://github.com/pingcap/tidb/pull/27859) - Fix the issue that Lightning connects to offline TiKV nodes during pre-check and causes import failures [#27826](https://github.com/pingcap/tidb/pull/27826) - Fix the issue that pre-check cost too much time when importing many files to tables [#27605](https://github.com/pingcap/tidb/issues/27605) @@ -376,7 +377,7 @@ Starting from TiCDC v5.3.0, the cyclic replication feature between TiDB clusters - Fix the issue that the scatter range scheduler cannot schedule empty Regions [#4118](https://github.com/tikv/pd/pull/4118) - Fix the issue that the key manager cost too much CPU [#4071](https://github.com/tikv/pd/issues/4071) - Fix the data race issue that might occur when setting configurations of hot Region scheduler [#4159](https://github.com/tikv/pd/issues/4159) - - Fix slow leader election caused by stucked Region syncer [#3936](https://github.com/tikv/pd/issues/3936) + - Fix slow leader election caused by stuck Region syncer [#3936](https://github.com/tikv/pd/issues/3936) + TiFlash diff --git a/releases/release-5.3.1.md b/releases/release-5.3.1.md index 09b3c1cf7b342..d586221c5cd40 100644 --- a/releases/release-5.3.1.md +++ b/releases/release-5.3.1.md @@ -1,5 +1,6 @@ --- title: TiDB 5.3.1 Release Notes +summary: TiDB 5.3.1 was released on March 3, 2022. The release includes compatibility changes, improvements, and bug fixes for TiDB, TiKV, PD, TiCDC, TiFlash, Backup & Restore (BR), and TiDB Data Migration (DM). Some notable changes include optimizing user login mode mapping, reducing TiCDC recovery time, and fixing various bugs in TiDB, TiKV, PD, TiFlash, and tools like TiCDC and TiDB Lightning. These fixes address issues related to data import, user login, garbage collection, configuration parameters, and more. --- # TiDB 5.3.1 Release Notes @@ -20,7 +21,7 @@ TiDB version: 5.3.1 - TiDB - - Optimize the mapping logic of user login mode to make the logging more MySQL-compatible [#30450](https://github.com/pingcap/tidb/issues/32648) + - Optimize the mapping logic of user login mode to make the logging more MySQL-compatible [#32648](https://github.com/pingcap/tidb/issues/32648) - TiKV @@ -142,7 +143,7 @@ TiDB version: 5.3.1 - Fix the issue that `stopped` changefeeds resume automatically after a cluster upgrade [#3473](https://github.com/pingcap/tiflow/issues/3473) - Fix the issue that default values cannot be replicated [#3793](https://github.com/pingcap/tiflow/issues/3793) - Fix the issue of overly frequent warnings caused by MySQL sink deadlock [#2706](https://github.com/pingcap/tiflow/issues/2706) - - Fix the bug that the `enable-old-value` configuration item is not automatically set to `true` on Canal and Maxwell protocols [#3676](https://github.com/pingcap/tiflow/issues/3676) + - Fix the bug that the `enable-old-value` configuration item is not automatically set to `true` on the Canal protocol [#3676](https://github.com/pingcap/tiflow/issues/3676) - Fix the issue that Avro sink does not support parsing JSON type columns [#3624](https://github.com/pingcap/tiflow/issues/3624) - Fix the negative value error in the changefeed checkpoint lag [#3010](https://github.com/pingcap/tiflow/issues/3010) diff --git a/releases/release-5.3.2.md b/releases/release-5.3.2.md index e78e7abbb8be4..2cfe4d3f85c8c 100644 --- a/releases/release-5.3.2.md +++ b/releases/release-5.3.2.md @@ -1,5 +1,6 @@ --- title: TiDB 5.3.2 Release Notes +summary: TiDB 5.3.2 was released on June 29, 2022. It is not recommended to use this version due to a known bug, which has been fixed in v5.3.3. The release includes compatibility changes, improvements, and bug fixes for TiDB, PD, TiKV, TiFlash, and various tools like TiDB Data Migration, TiDB Lightning, Backup & Restore, TiCDC, and TiDB Data Migration. --- # TiDB 5.3.2 Release Notes @@ -144,7 +145,7 @@ TiDB version: 5.3.2 - Fix the issue that TiCDC fails to start when the first PD set in `--pd` is not available after TLS is enabled [#4777](https://github.com/pingcap/tiflow/issues/4777) - Fix a bug that querying status through open API may be blocked when the PD node is abnormal [#4778](https://github.com/pingcap/tiflow/issues/4778) - Fix a stability problem in workerpool used by Unified Sorter [#4447](https://github.com/pingcap/tiflow/issues/4447) - - Fix a bug that sequence is incorrectly replicated in some cases [#4563](https://github.com/pingcap/tiflow/issues/4552) + - Fix a bug that sequence is incorrectly replicated in some cases [#4552](https://github.com/pingcap/tiflow/issues/4552) + TiDB Data Migration (DM) diff --git a/releases/release-5.3.3.md b/releases/release-5.3.3.md index f8cf5b9178546..e5c84650eb119 100644 --- a/releases/release-5.3.3.md +++ b/releases/release-5.3.3.md @@ -1,5 +1,6 @@ --- title: TiDB 5.3.3 Release Note +summary: TiDB 5.3.3 was released on September 14, 2022. The bug fix in TiKV addresses continuous SQL execution errors in the cluster after PD leader switch or PD restart. The issue was caused by a TiKV bug that has been fixed in v5.3.3. Affected versions include v5.3.2 and v5.4.2. Upgrading to v5.3.3 or restarting TiKV nodes can resolve the issue. For more details, refer to issue #12934 on GitHub. --- # TiDB 5.3.3 Release Note diff --git a/releases/release-5.3.4.md b/releases/release-5.3.4.md index ca20e7ca1c7d3..5addade1af19b 100644 --- a/releases/release-5.3.4.md +++ b/releases/release-5.3.4.md @@ -1,5 +1,6 @@ --- title: TiDB 5.3.4 Release Notes +summary: TiDB 5.3.4 was released on November 24, 2022. The release includes improvements to TiKV and bug fixes for TiDB, PD, TiFlash, Dumpling, and TiCDC. Some of the key bug fixes include issues related to TLS certificate reloading, Region cache cleanup, wrong data writing, database-level privileges, and authentication failures. Other fixes address issues with logical operators, stream timeout, leader switchover, and data dumping. --- # TiDB 5.3.4 Release Notes diff --git a/releases/release-5.4.0.md b/releases/release-5.4.0.md index 396693e162c44..832ac4884d262 100644 --- a/releases/release-5.4.0.md +++ b/releases/release-5.4.0.md @@ -1,5 +1,6 @@ --- title: TiDB 5.4 Release Notes +summary: TiDB 5.4 introduces support for the GBK character set, Index Merge, reading stale data, persisting statistics configuration, and using Raft Engine as the log storage engine of TiKV. It also improves backup impact, supports Azure Blob storage, and enhances TiFlash and the MPP engine. Compatibility changes include new system variables and configuration file parameters. Other improvements cover SQL, security, performance, stability, high availability, data migration, diagnostic efficiency, and deployment. Bug fixes address issues in TiDB, TiKV, PD, TiFlash, BR, TiCDC, DM, TiDB Lightning, and TiDB Binlog. --- # TiDB 5.4 Release Notes diff --git a/releases/release-5.4.1.md b/releases/release-5.4.1.md index 20cf54569ce46..6d14a10ccc17e 100644 --- a/releases/release-5.4.1.md +++ b/releases/release-5.4.1.md @@ -1,5 +1,6 @@ --- title: TiDB 5.4.1 Release Notes +summary: "TiDB 5.4.1 Release Notes: This release includes compatibility changes, improvements, and bug fixes for TiDB, TiKV, PD, TiFlash, and various tools. Improvements include support for using the PointGet plan, adding more logs and metrics, and displaying multiple Kubernetes clusters in the Grafana dashboard. Bug fixes address issues such as incorrect handling of date_format, wrong data writing, wrong query results, and various panics and errors. Fixes for TiKV, PD, TiFlash, and tools are also included." --- # TiDB 5.4.1 Release Notes @@ -144,7 +145,7 @@ TiDB v5.4.1 does not introduce any compatibility changes in product design. But - Fix incorrect metrics caused by owner changes [#4774](https://github.com/pingcap/tiflow/issues/4774) - Fix the TiCDC panic issue that might occur because `Canal-JSON` does not support nil [#4736](https://github.com/pingcap/tiflow/issues/4736) - Fix a stability problem in workerpool used by Unified Sorter [#4447](https://github.com/pingcap/tiflow/issues/4447) - - Fix a bug that sequence is incorrectly replicated in some cases [#4563](https://github.com/pingcap/tiflow/issues/4552) + - Fix a bug that sequence is incorrectly replicated in some cases [#4552](https://github.com/pingcap/tiflow/issues/4552) - Fix the TiCDC panic issue that might occur when `Canal-JSON` incorrectly handles `string` [#4635](https://github.com/pingcap/tiflow/issues/4635) - Fix a bug that a TiCDC node exits abnormally when a PD leader is killed [#4248](https://github.com/pingcap/tiflow/issues/4248) - Fix a bug that MySQL sink generates duplicated `replace` SQL statements when `batch-replace-enable` is disabled [#4501](https://github.com/pingcap/tiflow/issues/4501) diff --git a/releases/release-5.4.2.md b/releases/release-5.4.2.md index 963a4945753e5..2b9e8cc0aad16 100644 --- a/releases/release-5.4.2.md +++ b/releases/release-5.4.2.md @@ -1,5 +1,6 @@ --- title: TiDB 5.4.2 Release Notes +summary: TiDB 5.4.2 was released on July 8, 2022. It is not recommended to use this version due to a known bug, which has been fixed in v5.4.3. The release includes improvements to TiDB, TiKV, PD, and various tools, along with bug fixes for each component. These bug fixes address issues related to stability, performance, and error handling. --- # TiDB 5.4.2 Release Notes diff --git a/releases/release-5.4.3.md b/releases/release-5.4.3.md index e71ac559742f5..f23687bd3ad19 100644 --- a/releases/release-5.4.3.md +++ b/releases/release-5.4.3.md @@ -1,5 +1,6 @@ --- title: TiDB 5.4.3 Release Notes +summary: TiDB 5.4.3 was released on October 13, 2022. The release includes various improvements and bug fixes for TiKV, Tools, TiCDC, TiFlash, PD, and other tools. Improvements include support for configuring RocksDB write stall settings, optimizing Scatter Region to batch mode, and reducing performance overhead in multi-Region scenarios. Bug fixes address issues such as incorrect output of `SHOW CREATE PLACEMENT POLICY`, DDL statements getting stuck after PD node replacement, and various issues causing incorrect results and errors in TiDB, TiKV, PD, TiFlash, and other tools. The release also provides workarounds and affected versions for specific issues. --- # TiDB 5.4.3 Release Notes @@ -75,7 +76,7 @@ TiDB version: 5.4.3 + TiDB Lightning - - Fix the issue that an auto-increment column of the `BIGINT` type might be out of range [#27397](https://github.com/pingcap/tidb/issues/27937) + - Fix the issue that an auto-increment column of the `BIGINT` type might be out of range [#27937](https://github.com/pingcap/tidb/issues/27937) - Fix the issue that de-duplication might cause TiDB Lightning to panic in extreme cases [#34163](https://github.com/pingcap/tidb/issues/34163) - Fix the issue that TiDB Lightning does not support columns starting with slash, number, or non-ascii characters in Parquet files [#36980](https://github.com/pingcap/tidb/issues/36980) - Fix the issue that TiDB Lightning fails to connect to TiDB when TiDB uses an IPv6 host [#35880](https://github.com/pingcap/tidb/issues/35880) diff --git a/releases/release-6.0.0-dmr.md b/releases/release-6.0.0-dmr.md index 73198ae4f28a6..8ceb69dcc9b65 100644 --- a/releases/release-6.0.0-dmr.md +++ b/releases/release-6.0.0-dmr.md @@ -1,5 +1,6 @@ --- title: TiDB 6.0.0 Release Notes +summary: Learn about the new features, compatibility changes, improvements, and bug fixes in TiDB 6.0.0. --- # TiDB 6.0.0 Release Notes @@ -270,7 +271,7 @@ TiDB v6.0.0 is a DMR, and its version is 6.0.0-DMR. TiEM not only provides full lifecycle visual management for TiDB clusters, but also provides one-stop services: parameter management, version upgrades, cluster clone, active-standby cluster switching, data import and export, data replication, and data backup and restore services. TiEM can improve the efficiency of DevOps on TiDB and reduce the DevOps cost for enterprises. - Currently, TiEM is provided in the [TiDB Enterprise](https://en.pingcap.com/tidb-enterprise/) edition only. To get TiEM, contact us via the [TiDB Enterprise](https://en.pingcap.com/tidb-enterprise/) page. + Currently, TiEM is provided in the [TiDB Enterprise](https://www.pingcap.com/tidb-enterprise/) edition only. To get TiEM, contact us via the [TiDB Enterprise](https://www.pingcap.com/tidb-enterprise/) page. - Support customizing configurations of the monitoring components @@ -555,7 +556,7 @@ TiDB v6.0.0 is a DMR, and its version is 6.0.0-DMR. - A `loader..on-duplicate` parameter is added. The default value is `replace`, which means using the new data to replace the existing data. If you want to keep the previous behavior, you can set the value to `error`. This parameter only controls the behavior during the full import phase. - To use DM, you should use the corresponding version of `dmctl` - Due to internal mechanism changes, after upgrading DM to v6.0.0, you should also upgrade `dmctl` to v6.0.0. -- In v5.4 (v5.4 only), TiDB allows incorrect values for some noop system variables. Since v6.0.0, TiDB disallows setting incorrect values for system variables. [#31538](https://github.com/pingcap/tidb/issues/31538) +- For v5.4 and earlier versions, TiDB allows incorrect values for some noop system variables. Starting from v6.0.0, TiDB disallows setting incorrect values for system variables. [#31538](https://github.com/pingcap/tidb/issues/31538) ## Improvements @@ -764,7 +765,7 @@ TiDB v6.0.0 is a DMR, and its version is 6.0.0-DMR. - Fix a bug that a TiCDC node exits abnormally when a PD leader is killed [#4248](https://github.com/pingcap/tiflow/issues/4248) - Fix the error `Unknown system variable 'transaction_isolation'` for some MySQL versions [#4504](https://github.com/pingcap/tiflow/issues/4504) - Fix the TiCDC panic issue that might occur when `Canal-JSON` incorrectly handles `string` [#4635](https://github.com/pingcap/tiflow/issues/4635) - - Fix a bug that sequence is incorrectly replicated in some cases [#4563](https://github.com/pingcap/tiflow/issues/4552) + - Fix a bug that sequence is incorrectly replicated in some cases [#4552](https://github.com/pingcap/tiflow/issues/4552) - Fix the TiCDC panic issue that might occur because `Canal-JSON` does not support nil [#4736](https://github.com/pingcap/tiflow/issues/4736) - Fix the wrong data mapping for avro codec of type `Enum/Set` and `TinyText/MediumText/Text/LongText` [#4454](https://github.com/pingcap/tiflow/issues/4454) - Fix a bug that Avro converts a `NOT NULL` column to a nullable field [#4818](https://github.com/pingcap/tiflow/issues/4818) diff --git a/releases/release-6.1.0.md b/releases/release-6.1.0.md index 4b555be897299..c612f4d0f0260 100644 --- a/releases/release-6.1.0.md +++ b/releases/release-6.1.0.md @@ -9,7 +9,7 @@ Release date: June 13, 2022 TiDB version: 6.1.0 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.1.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) In 6.1.0, the key new features or improvements are as follows: @@ -59,11 +59,17 @@ In 6.1.0, the key new features or improvements are as follows: ### Performance -* Support customized Region size (experimental) +* Support customized Region size - Setting Regions to a larger size can effectively reduce the number of Regions, make Regions easier to manage, and improve the cluster performance and stability. This feature introduces the concept of bucket, which is a smaller range within a Region. Using buckets as the query unit can optimize concurrent query performance when Regions are set to a larger size. Using buckets as the query unit can also dynamically adjust the sizes of hot Regions to ensure the scheduling efficiency and load balance. This feature is currently experimental. It is not recommended to use it in production environments. + Starting from v6.1.0, you can configure [`coprocessor.region-split-size`](/tikv-configuration-file.md#region-split-size) to set Regions to a larger size. This can effectively reduce the number of Regions, make Regions easier to manage, and improve the cluster performance and stability. - [User document](/tune-region-performance.md), [#11515](https://github.com/tikv/tikv/issues/11515) + [User document](/tune-region-performance.md#use-region-split-size-to-adjust-region-size), [#11515](https://github.com/tikv/tikv/issues/11515) + +* Support using buckets to increase concurrency (experimental) + + To help you further improve the query concurrency after setting Regions to a larger size, TiDB introduces the concept of bucket, which is a smaller range within a Region. Using buckets as the query unit can optimize concurrent query performance when Regions are set to a larger size. Using buckets as the query unit can also dynamically adjust the sizes of hotspot Regions to ensure the scheduling efficiency and load balance. This feature is currently experimental. It is not recommended to use it in production environments. + + [User document](/tune-region-performance.md#use-bucket-to-increase-concurrency), [#11515](https://github.com/tikv/tikv/issues/11515) * Use Raft Engine as the default log storage engine @@ -336,7 +342,7 @@ In 6.1.0, the key new features or improvements are as follows: - CDC supports RawKV [#11965](https://github.com/tikv/tikv/issues/11965) - Support splitting a large snapshot file into multiple files [#11595](https://github.com/tikv/tikv/issues/11595) - Move the snapshot garbage collection from Raftstore to background thread to prevent snapshot GC from blocking Raftstore message loops [#11966](https://github.com/tikv/tikv/issues/11966) - - Support dynamic setting of the the maximum message length (`max-grpc-send-msg-len`) and the maximum batch size of gPRC messages (`raft-msg-max-batch-size`) [#12334](https://github.com/tikv/tikv/issues/12334) + - Support dynamic setting of the maximum message length (`max-grpc-send-msg-len`) and the maximum batch size of gPRC messages (`raft-msg-max-batch-size`) [#12334](https://github.com/tikv/tikv/issues/12334) - Support executing online unsafe recovery plan through Raft [#10483](https://github.com/tikv/tikv/issues/10483) + PD @@ -412,7 +418,7 @@ In 6.1.0, the key new features or improvements are as follows: + TiDB Data Migration (DM) - - Fix the `start-time` time zone issue and change DM behavior from using the downstream time zone to using the upstream time zone [#5271](https://github.com/pingcap/tiflow/issues/5471) + - Fix the `start-time` time zone issue and change DM behavior from using the downstream time zone to using the upstream time zone [#5471](https://github.com/pingcap/tiflow/issues/5471) - Fix the issue that DM occupies more disk space after the task automatically resumes [#3734](https://github.com/pingcap/tiflow/issues/3734) [#5344](https://github.com/pingcap/tiflow/issues/5344) - Fix the problem that checkpoint flush may cause the data of failed rows to be skipped [#5279](https://github.com/pingcap/tiflow/issues/5279) - Fix the issue that in some cases manually executing the filtered DDL in the downstream might cause task resumption failure [#5272](https://github.com/pingcap/tiflow/issues/5272) @@ -428,5 +434,5 @@ In 6.1.0, the key new features or improvements are as follows: - Fix the issue that the precheck does not check local disk resources and cluster availability [#34213](https://github.com/pingcap/tidb/issues/34213) - Fix the issue of incorrect routing for schemas [#33381](https://github.com/pingcap/tidb/issues/33381) - Fix the issue that the PD configuration is not restored correctly when TiDB Lightning panics [#31733](https://github.com/pingcap/tidb/issues/31733) - - Fix the issue of Local-backend import failure caused by out-of-bounds data in the `auto_increment` column [#29737](https://github.com/pingcap/tidb/issues/27937) + - Fix the issue of Local-backend import failure caused by out-of-bounds data in the `auto_increment` column [#27937](https://github.com/pingcap/tidb/issues/27937) - Fix the issue of local backend import failure when the `auto_random` or `auto_increment` column is null [#34208](https://github.com/pingcap/tidb/issues/34208) diff --git a/releases/release-6.1.1.md b/releases/release-6.1.1.md index 4842ca74d53b6..0278613f7d582 100644 --- a/releases/release-6.1.1.md +++ b/releases/release-6.1.1.md @@ -1,5 +1,6 @@ --- title: TiDB 6.1.1 Release Notes +summary: TiDB 6.1.1 was released on September 1, 2022. Changes include case-insensitive `SHOW DATABASES LIKE` statement, default value change for `tidb_enable_outer_join_reorder`, and improvements in optimizer and metrics response compression. Bug fixes address issues such as hanging `INL_HASH_JOIN`, panicking during `UPDATE` statement execution, and incorrect query results. Other changes include multi-level support for different quality standards and additions to the `TiDB-community-toolkit` binary package. --- # TiDB 6.1.1 Release Notes @@ -8,7 +9,7 @@ Release date: September 1, 2022 TiDB version: 6.1.1 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.1.1#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) ## Compatibility changes @@ -80,7 +81,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with- - Fix the issue that the wrong join reorder in some right outer join scenarios causes wrong query result [#36912](https://github.com/pingcap/tidb/issues/36912) @[winoros](https://github.com/winoros) - Fix the issue of incorrectly inferred null flag of the TiFlash `firstrow` aggregate function in the EqualAll case [#34584](https://github.com/pingcap/tidb/issues/34584) @[fixdb](https://github.com/fixdb) - Fix the issue that Plan Cache does not work when a binding is created with the `IGNORE_PLAN_CACHE` hint [#34596](https://github.com/pingcap/tidb/issues/34596) @[fzzf678](https://github.com/fzzf678) - - Fix the issu that an `EXCHANGE` operator is missing between the hash-partition window and the single-partition window [#35990](https://github.com/pingcap/tidb/issues/35990) @[LittleFall](https://github.com/LittleFall) + - Fix the issue that an `EXCHANGE` operator is missing between the hash-partition window and the single-partition window [#35990](https://github.com/pingcap/tidb/issues/35990) @[LittleFall](https://github.com/LittleFall) - Fix the issue that partitioned tables cannot fully use indexes to scan data in some cases [#33966](https://github.com/pingcap/tidb/issues/33966) @[mjonss](https://github.com/mjonss) - Fix the issue of wrong query result when a wrong default value is set for partial aggregation after the aggregation is pushed down [#35295](https://github.com/pingcap/tidb/issues/35295) @[tiancaiamao](https://github.com/tiancaiamao) - Fix the issue that querying partitioned tables might get the `index-out-of-range` error in some cases [#35181](https://github.com/pingcap/tidb/issues/35181) @[mjonss](https://github.com/mjonss) @@ -151,16 +152,16 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with- + TiCDC - - Fix the wrong maximum compatible version number [#6039](https://github.com/pingcap/tiflow/issues/6039) @[hi-rustin](https://github.com/hi-rustin) + - Fix the wrong maximum compatible version number [#6039](https://github.com/pingcap/tiflow/issues/6039) @[hi-rustin](https://github.com/Rustin170506) - Fix a bug that may cause the cdc server to panic when it receives an HTTP request before it fully starts [#5639](https://github.com/pingcap/tiflow/issues/5639) @[asddongmen](https://github.com/asddongmen) - Fix the ddl sink panic issue when the changefeed sync-point is enabled [#4934](https://github.com/pingcap/tiflow/issues/4934) @[asddongmen](https://github.com/asddongmen) - Fix the issue that a changefeed is stuck in some scenarios when sync-point is enabled [#6827](https://github.com/pingcap/tiflow/issues/6827) @[hicqu](https://github.com/hicqu) - Fix a bug that changefeed API does not work properly after the cdc server restarts [#5837](https://github.com/pingcap/tiflow/issues/5837) @[asddongmen](https://github.com/asddongmen) - Fix the data race issue in the black hole sink [#6206](https://github.com/pingcap/tiflow/issues/6206) @[asddongmen](https://github.com/asddongmen) - - Fix the TiCDC panic issue when you set `enable-old-value = false` [#6198](https://github.com/pingcap/tiflow/issues/6198) @[hi-rustin](https://github.com/hi-rustin) + - Fix the TiCDC panic issue when you set `enable-old-value = false` [#6198](https://github.com/pingcap/tiflow/issues/6198) @[hi-rustin](https://github.com/Rustin170506) - Fix the data consistency issue when the redo log feature is enabled [#6189](https://github.com/pingcap/tiflow/issues/6189) [#6368](https://github.com/pingcap/tiflow/issues/6368) [#6277](https://github.com/pingcap/tiflow/issues/6277) [#6456](https://github.com/pingcap/tiflow/issues/6456) [#6695](https://github.com/pingcap/tiflow/issues/6695) [#6764](https://github.com/pingcap/tiflow/issues/6764) [#6859](https://github.com/pingcap/tiflow/issues/6859) @[asddongmen](https://github.com/asddongmen) - Fix poor redo log performance by writing redo events asynchronously [#6011](https://github.com/pingcap/tiflow/issues/6011) @[CharlesCheung96](https://github.com/CharlesCheung96) - - Fix the issue that the MySQL sink cannot connect to IPv6 addresses [#6135](https://github.com/pingcap/tiflow/issues/6135) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that the MySQL sink cannot connect to IPv6 addresses [#6135](https://github.com/pingcap/tiflow/issues/6135) @[hi-rustin](https://github.com/Rustin170506) + Backup & Restore (BR) diff --git a/releases/release-6.1.2.md b/releases/release-6.1.2.md index 5de6535239c9e..89a1bb57d12b2 100644 --- a/releases/release-6.1.2.md +++ b/releases/release-6.1.2.md @@ -1,5 +1,6 @@ --- title: TiDB 6.1.2 Release Notes +summary: TiDB 6.1.2 was released on October 24, 2022. The release includes improvements to TiDB, TiKV, Tools, PD, TiFlash, and bug fixes for various issues in each component. The improvements include setting placement rules and TiFlash replicas simultaneously, support for configuring various settings, and enhancing performance. Bug fixes address issues such as incorrect cleanup of privileges, incorrect output, query failures, and performance issues. --- # TiDB 6.1.2 Release Notes @@ -8,7 +9,7 @@ Release date: October 24, 2022 TiDB version: 6.1.2 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.1.2#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) ## Improvements @@ -85,9 +86,9 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with- + TiCDC - Fix the issue that the cdc server might panic if it receives an HTTP request before the cdc server fully starts [#6838](https://github.com/pingcap/tiflow/issues/6838) @[asddongmen](https://github.com/asddongmen) - - Fix the log flooding issue during upgrade [#7235](https://github.com/pingcap/tiflow/issues/7235) @[hi-rustin](https://github.com/hi-rustin) - - Fix the issue that changefeed's redo log files might be deleted by mistake [#6413](https://github.com/pingcap/tiflow/issues/6413) @[hi-rustin](https://github.com/hi-rustin) - - Fix the issue that TiCDC might become unavailable when too many operations in an etcd transaction are committed [#7131](https://github.com/pingcap/tiflow/issues/7131) @[hi-rustin](https://github.com/hi-rustin) + - Fix the log flooding issue during upgrade [#7235](https://github.com/pingcap/tiflow/issues/7235) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that changefeed's redo log files might be deleted by mistake [#6413](https://github.com/pingcap/tiflow/issues/6413) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that TiCDC might become unavailable when too many operations in an etcd transaction are committed [#7131](https://github.com/pingcap/tiflow/issues/7131) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that data inconsistency might occur when non-reentrant DDL statements in redo logs are executed twice [#6927](https://github.com/pingcap/tiflow/issues/6927) @[hicqu](https://github.com/hicqu) + Backup & Restore (BR) diff --git a/releases/release-6.1.3.md b/releases/release-6.1.3.md index 45aa0fe4f3ca8..2fa7956f4be62 100644 --- a/releases/release-6.1.3.md +++ b/releases/release-6.1.3.md @@ -1,5 +1,6 @@ --- title: TiDB 6.1.3 Release Notes +summary: TiDB 6.1.3 was released on December 5, 2022. The release includes compatibility changes, improvements, bug fixes, and updates to various tools such as TiCDC, PD, TiKV, TiFlash, Backup & Restore, TiCDC, and TiDB Data Migration. Some notable changes include default value changes in TiCDC, lock granularity optimization in PD, and bug fixes in TiDB, PD, TiKV, TiFlash, and various tools. The release also includes an upgrade to the Go compiler version of TiDB from go1.18 to go1.19, which improves stability. --- # TiDB 6.1.3 Release Notes @@ -8,7 +9,7 @@ Release date: December 5, 2022 TiDB version: 6.1.3 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.1.3#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) ## Compatibility changes diff --git a/releases/release-6.1.4.md b/releases/release-6.1.4.md index 7dc2dc3ed32d5..2d05f02c3e32b 100644 --- a/releases/release-6.1.4.md +++ b/releases/release-6.1.4.md @@ -9,7 +9,7 @@ Release date: February 8, 2023 TiDB version: 6.1.4 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.1.4#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) ## Compatibility changes @@ -74,10 +74,10 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with- + TiCDC - Fix the issue that the checkpoint cannot advance when TiCDC replicates an excessively large number of tables [#8004](https://github.com/pingcap/tiflow/issues/8004) @[asddongmen](https://github.com/asddongmen) - - Fix the issue that `transaction_atomicity` and `protocol` cannot be updated via the configuration file [#7935](https://github.com/pingcap/tiflow/issues/7935) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that `transaction-atomicity` and `protocol` cannot be updated via the configuration file [#7935](https://github.com/pingcap/tiflow/issues/7935) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the issue that TiCDC mistakenly reports an error when the version of TiFlash is later than that of TiCDC [#7744](https://github.com/pingcap/tiflow/issues/7744) @[overvenus](https://github.com/overvenus) - Fix the issue that OOM occurs when TiCDC replicates large transactions [#7913](https://github.com/pingcap/tiflow/issues/7913) @[overvenus](https://github.com/overvenus) - - Fix a bug that the context deadline is exceeded when TiCDC replicates data without splitting large transactions [#7982](https://github.com/pingcap/tiflow/issues/7982) @[hi-rustin](https://github.com/hi-rustin) + - Fix a bug that the context deadline is exceeded when TiCDC replicates data without splitting large transactions [#7982](https://github.com/pingcap/tiflow/issues/7982) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that `sasl-password` in the `changefeed query` result is not masked [#7182](https://github.com/pingcap/tiflow/issues/7182) @[dveeden](https://github.com/dveeden) - Fix the issue that data is lost when a user quickly deletes a replication task and then creates another one with the same task name [#7657](https://github.com/pingcap/tiflow/issues/7657) @[overvenus](https://github.com/overvenus) diff --git a/releases/release-6.1.5.md b/releases/release-6.1.5.md index 846ffe4e056db..7d72630a47681 100644 --- a/releases/release-6.1.5.md +++ b/releases/release-6.1.5.md @@ -9,7 +9,7 @@ Release date: February 28, 2023 TiDB version: 6.1.5 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.1.5#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) ## Compatibility changes diff --git a/releases/release-6.1.6.md b/releases/release-6.1.6.md index c21c534b24fd7..17725f58ad6e3 100644 --- a/releases/release-6.1.6.md +++ b/releases/release-6.1.6.md @@ -9,7 +9,7 @@ Release date: April 12, 2023 TiDB version: 6.1.6 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.1.6#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) ## Compatibility changes @@ -92,7 +92,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with- + TiDB Lightning - - Fix the issue that the conflict resolution logic (`duplicate-resolution`) might lead to inconsistent checksums [#40657](https://github.com/pingcap/tidb/issues/40657) @[gozssky](https://github.com/gozssky) + - Fix the issue that the conflict resolution logic (`duplicate-resolution`) might lead to inconsistent checksums [#40657](https://github.com/pingcap/tidb/issues/40657) @[sleepymole](https://github.com/sleepymole) - Fix the issue that TiDB Lightning panics in the split-region phase [#40934](https://github.com/pingcap/tidb/issues/40934) @[lance6716](https://github.com/lance6716) - Fix the issue that when importing data in Local Backend mode, the target columns do not automatically generate data if the compound primary key of the imported target table has an `auto_random` column and no value for the column is specified in the source data [#41454](https://github.com/pingcap/tidb/issues/41454) @[D3Hunter](https://github.com/D3Hunter) - Fix the issue that TiDB Lightning might incorrectly skip conflict resolution when all but the last TiDB Lightning instance encounters a local duplicate record during a parallel import [#40923](https://github.com/pingcap/tidb/issues/40923) @[lichunzhu](https://github.com/lichunzhu) diff --git a/releases/release-6.1.7.md b/releases/release-6.1.7.md index 728ea0b6dab7d..1ec99802e66d8 100644 --- a/releases/release-6.1.7.md +++ b/releases/release-6.1.7.md @@ -9,7 +9,7 @@ Release date: July 12, 2023 TiDB version: 6.1.7 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.1.7#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) ## Improvements @@ -88,9 +88,9 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with- + TiCDC - - Fix the issue that TiCDC cannot create a changefeed with a downstream Kafka-on-Pulsar [#8892](https://github.com/pingcap/tiflow/issues/8892) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that TiCDC cannot create a changefeed with a downstream Kafka-on-Pulsar [#8892](https://github.com/pingcap/tiflow/issues/8892) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that TiCDC cannot automatically recover when PD address or leader fails [#8812](https://github.com/pingcap/tiflow/issues/8812) [#8877](https://github.com/pingcap/tiflow/issues/8877) @[asddongmen](https://github.com/asddongmen) - - Fix the issue that when the downstream is Kafka, TiCDC queries the downstream metadata too frequently and causes excessive workload in the downstream [#8957](https://github.com/pingcap/tiflow/issues/8957) [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that when the downstream is Kafka, TiCDC queries the downstream metadata too frequently and causes excessive workload in the downstream [#8957](https://github.com/pingcap/tiflow/issues/8957) [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that TiCDC gets stuck when PD fails such as network isolation or PD Owner node reboot [#8808](https://github.com/pingcap/tiflow/issues/8808) [#8812](https://github.com/pingcap/tiflow/issues/8812) [#8877](https://github.com/pingcap/tiflow/issues/8877) @[asddongmen](https://github.com/asddongmen) + TiDB Lightning diff --git a/releases/release-6.2.0.md b/releases/release-6.2.0.md index 73893616d5c4a..5279f419fac43 100644 --- a/releases/release-6.2.0.md +++ b/releases/release-6.2.0.md @@ -1,5 +1,6 @@ --- title: TiDB 6.2.0 Release Notes +summary: TiDB 6.2.0-DMR introduces new features like visual execution plans, monitoring page, and lock view. It also supports concurrent DDL operations and enhances the performance of aggregation operations. TiKV now supports automatic CPU usage tuning and detailed configuration information listing. TiFlash adds FastScan for data scanning and improves error handling. BR now supports continuous data validation and automatically identifies the region of Amazon S3 buckets. TiCDC supports filtering DDL and DML events. There are also compatibility changes, bug fixes, and improvements across various tools. --- # TiDB 6.2.0 Release Notes @@ -221,7 +222,7 @@ In v6.2.0-DMR, the key new features and improvements are as follows: This feature does not need manual configuration. If your TiDB cluster is v6.1.0 or later versions and TiDB Lightning is v6.2.0 or later versions, the new physical import mode takes effect automatically. - [User document](/tidb-lightning/tidb-lightning-physical-import-mode-usage.md#scope-of-pausing-scheduling-during-import) [#35148](https://github.com/pingcap/tidb/issues/35148) @[gozssky](https://github.com/gozssky) + [User document](/tidb-lightning/tidb-lightning-physical-import-mode-usage.md#scope-of-pausing-scheduling-during-import) [#35148](https://github.com/pingcap/tidb/issues/35148) @[sleepymole](https://github.com/sleepymole) * Refactor the [user documentation of TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md) to make its structure more reasonable and clear. The terms for "backend" is also modified to lower the understanding barrier for new users: @@ -260,7 +261,7 @@ In v6.2.0-DMR, the key new features and improvements are as follows: | [tidb_enable_noop_variables](/system-variables.md#tidb_enable_noop_variables-new-in-v620) | Newly added | This variable controls whether to show `noop` variables in the result of `SHOW [GLOBAL] VARIABLES`. | | [tidb_min_paging_size](/system-variables.md#tidb_min_paging_size-new-in-v620) | Newly added | This variable is used to set the maximum number of rows during the coprocessor paging request process. | | [tidb_txn_commit_batch_size](/system-variables.md#tidb_txn_commit_batch_size-new-in-v620) | Newly added | This variable is used to control the batch size of transaction commit requests that TiDB sends to TiKV. | -| tidb_enable_change_multi_schema | Deleted | This variable is used to control whether multiple columns or indexes can be altered in one `ALTER TABLE` statement. | +| tidb_enable_change_multi_schema | Deleted | This variable is deleted because, starting from v6.2.0, you can alter multiple columns or indexes in one `ALTER TABLE` statement by default. | | [tidb_enable_outer_join_reorder](/system-variables.md#tidb_enable_outer_join_reorder-new-in-v610) | Modified | This variable controls whether the Join Reorder algorithm of TiDB supports Outer Join. In v6.1.0, the default value is `ON`, which means the Join Reorder's support for Outer Join is enabled by default. From v6.2.0, the default value is `OFF`, which means the support is disabled by default. | ### Configuration file parameters @@ -328,7 +329,7 @@ Since TiDB v6.2.0, backing up and restoring RawKV using BR is deprecated. - Support the `SHOW COUNT(*) WARNINGS` and `SHOW COUNT(*) ERRORS` statements [#25068](https://github.com/pingcap/tidb/issues/25068) @[likzn](https://github.com/likzn) - Add validation check for some system variables [#35048](https://github.com/pingcap/tidb/issues/35048) @[morgo](https://github.com/morgo) - - Optimize the error messages for some type conversions [#32447](https://github.com/pingcap/tidb/issues/32744) @[fanrenhoo](https://github.com/fanrenhoo) + - Optimize the error messages for some type conversions [#32744](https://github.com/pingcap/tidb/issues/32744) @[fanrenhoo](https://github.com/fanrenhoo) - The `KILL` command now supports DDL operations [#24144](https://github.com/pingcap/tidb/issues/24144) @[morgo](https://github.com/morgo) - Make the output of `SHOW TABLES/DATABASES LIKE …` more MySQL-compatible. The column names in the output contain the `LIKE` value [#35116](https://github.com/pingcap/tidb/issues/35116) @[likzn](https://github.com/likzn) - Improve the performance of JSON-related functions [#35859](https://github.com/pingcap/tidb/issues/35859) @[wjhuang2016](https://github.com/wjhuang2016) diff --git a/releases/release-6.3.0.md b/releases/release-6.3.0.md index 517ddc6b08c55..f1644233192db 100644 --- a/releases/release-6.3.0.md +++ b/releases/release-6.3.0.md @@ -1,5 +1,6 @@ --- title: TiDB 6.3.0 Release Notes +summary: TiDB 6.3.0-DMR, released on September 30, 2022, introduces new features and improvements, including encryption at rest using the SM4 algorithm in TiKV, authentication using the SM3 algorithm in TiDB, and support for JSON data type and functions. It also provides execution time metrics at a finer granularity, enhances output for slow logs and `TRACE` statements, and supports deadlock history information in TiDB Dashboard. Additionally, TiDB v6.3.0 introduces new system variables and configuration file parameters, and fixes various bugs and issues. The release also includes improvements in TiKV, PD, TiFlash, Backup & Restore (BR), TiCDC, TiDB Binlog, TiDB Data Migration (DM), and TiDB Lightning. --- # TiDB 6.3.0 Release Notes @@ -12,7 +13,7 @@ TiDB version: 6.3.0-DMR > > The TiDB 6.3.0-DMR documentation has been [archived](https://docs-archive.pingcap.com/tidb/v6.3/). PingCAP encourages you to use [the latest LTS version](https://docs.pingcap.com/tidb/stable) of the TiDB database. -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.3/quick-start-with-tidb) | [Installation packages](https://www.pingcap.com/download/?version=v6.3.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.3/quick-start-with-tidb) In v6.3.0-DMR, the key new features and improvements are as follows: @@ -316,8 +317,8 @@ Since v6.3.0, TiCDC no longer supports configuring Pulsar sink. [kop](https://gi - Improve TiCDC's compatibility with the concurrent DDL framework introduced in the upstream TiDB [#6506](https://github.com/pingcap/tiflow/issues/6506) @[lance6716](https://github.com/lance6716) - Support logging `start ts` of DML statements when MySQL sink gets an error [#6460](https://github.com/pingcap/tiflow/issues/6460) @[overvenus](https://github.com/overvenus) - Enhance the `api/v1/health` API to return a more accurate health state of a TiCDC cluster [#4757](https://github.com/pingcap/tiflow/issues/4757) @[overvenus](https://github.com/overvenus) - - Implement MQ sink and MySQL sink in the asynchronous mode to improve the sink throughput [#5928](https://github.com/pingcap/tiflow/issues/5928) @[hicqu](https://github.com/hicqu) @[hi-rustin](https://github.com/hi-rustin) - - Delete the deprecated pulsar sink [#7087](https://github.com/pingcap/tiflow/issues/7087) @[hi-rustin](https://github.com/hi-rustin) + - Implement MQ sink and MySQL sink in the asynchronous mode to improve the sink throughput [#5928](https://github.com/pingcap/tiflow/issues/5928) @[hicqu](https://github.com/hicqu) @[hi-rustin](https://github.com/Rustin170506) + - Delete the deprecated pulsar sink [#7087](https://github.com/pingcap/tiflow/issues/7087) @[hi-rustin](https://github.com/Rustin170506) - Improve replication performance by discarding DDL statements that are irrelevant to a changefeed [#6447](https://github.com/pingcap/tiflow/issues/6447) @[asddongmen](https://github.com/asddongmen) + TiDB Data Migration (DM) @@ -335,7 +336,7 @@ Since v6.3.0, TiCDC no longer supports configuring Pulsar sink. [kop](https://gi - Fix the issue that the privilege check is skipped for `PREPARE` statements [#35784](https://github.com/pingcap/tidb/issues/35784) @[lcwangchao](https://github.com/lcwangchao) - Fix the issue that the system variable `tidb_enable_noop_variable` can be set to `WARN` [#36647](https://github.com/pingcap/tidb/issues/36647) @[lcwangchao](https://github.com/lcwangchao) - - Fix the issue that when an expression index is defined, the `ORDINAL_POSITION` column of the `INFORMAITON_SCHEMA.COLUMNS` table might be incorrect [#31200](https://github.com/pingcap/tidb/issues/31200) @[bb7133](https://github.com/bb7133) + - Fix the issue that when an expression index is defined, the `ORDINAL_POSITION` column of the `INFORMATION_SCHEMA.COLUMNS` table might be incorrect [#31200](https://github.com/pingcap/tidb/issues/31200) @[bb7133](https://github.com/bb7133) - Fix the issue that TiDB does not report an error when the timestamp is larger than `MAXINT32` [#31585](https://github.com/pingcap/tidb/issues/31585) @[bb7133](https://github.com/bb7133) - Fix the issue that TiDB server cannot be started when the enterprise plugin is used [#37319](https://github.com/pingcap/tidb/issues/37319) @[xhebox](https://github.com/xhebox) - Fix the incorrect output of `SHOW CREATE PLACEMENT POLICY` [#37526](https://github.com/pingcap/tidb/issues/37526) @[xhebox](https://github.com/xhebox) @@ -353,7 +354,7 @@ Since v6.3.0, TiCDC no longer supports configuring Pulsar sink. [kop](https://gi - Fix the issue that the cast and comparison between binary strings and JSON in TiDB are incompatible with MySQL [#31918](https://github.com/pingcap/tidb/issues/31918) [#25053](https://github.com/pingcap/tidb/issues/25053) @[YangKeao](https://github.com/YangKeao) - Fix the issue that `JSON_OBJECTAGG` and `JSON_ARRAYAGG` in TiDB are not compatible with MySQL on binary values [#25053](https://github.com/pingcap/tidb/issues/25053) @[YangKeao](https://github.com/YangKeao) - Fix the issue that the comparison between JSON opaque values causes panic [#37315](https://github.com/pingcap/tidb/issues/37315) @[YangKeao](https://github.com/YangKeao) - - Fix the issue that the single precision float cannot be used in JSON aggregation funtions [#37287](https://github.com/pingcap/tidb/issues/37287) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that the single precision float cannot be used in JSON aggregation functions [#37287](https://github.com/pingcap/tidb/issues/37287) @[YangKeao](https://github.com/YangKeao) - Fix the issue that the `UNION` operator might return unexpected empty result [#36903](https://github.com/pingcap/tidb/issues/36903) @[tiancaiamao](https://github.com/tiancaiamao) - Fix the issue that the result of the `castRealAsTime` expression is inconsistent with MySQL [#37462](https://github.com/pingcap/tidb/issues/37462) @[mengxin9014](https://github.com/mengxin9014) - Fix the issue that pessimistic DML operations lock non-unique index keys [#36235](https://github.com/pingcap/tidb/issues/36235) @[ekexium](https://github.com/ekexium) diff --git a/releases/release-6.4.0.md b/releases/release-6.4.0.md index 98e1499960f1a..1984ea9e849f0 100644 --- a/releases/release-6.4.0.md +++ b/releases/release-6.4.0.md @@ -1,5 +1,6 @@ --- title: TiDB 6.4.0 Release Notes +summary: TiDB 6.4.0-DMR introduces new features and improvements, including support for restoring a cluster to a specific point in time, compatibility with Linear Hash partitioning syntax, and a high-performance `AUTO_INCREMENT` mode. It also enhances fault recovery, memory usage control, and statistics collection. TiFlash now supports the SM4 algorithm for encryption at rest, and TiCDC supports replicating data to Kafka. The release also includes bug fixes and improvements across various tools and components. --- # TiDB 6.4.0 Release Notes @@ -12,11 +13,11 @@ TiDB version: 6.4.0-DMR > > The TiDB 6.4.0-DMR documentation has been [archived](https://docs-archive.pingcap.com/tidb/v6.4/). PingCAP encourages you to use [the latest LTS version](https://docs.pingcap.com/tidb/stable) of the TiDB database. -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.4/quick-start-with-tidb) | [Installation packages](https://www.pingcap.com/download/?version=v6.4.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.4/quick-start-with-tidb) In v6.4.0-DMR, the key new features and improvements are as follows: -- Support restoring a cluster to a specific point in time by using [`FLASHBACK CLUSTER TO TIMESTAMP`](/sql-statements/sql-statement-flashback-to-timestamp.md) (experimental). +- Support restoring a cluster to a specific point in time by using [`FLASHBACK CLUSTER TO TIMESTAMP`](/sql-statements/sql-statement-flashback-cluster.md) (experimental). - Support [tracking the global memory usage](/configure-memory-usage.md) of TiDB instances (experimental). - Be compatible with [the Linear Hash partitioning syntax](/partitioned-table.md#how-tidb-handles-linear-hash-partitions). - Support a high-performance and globally monotonic [`AUTO_INCREMENT`](/auto-increment.md#mysql-compatibility-mode) (experimental). @@ -48,7 +49,7 @@ In v6.4.0-DMR, the key new features and improvements are as follows: Before executing `FLASHBACK CLUSTER TO TIMESTAMP`, you need to pause PITR and replication tasks running on such tools as TiCDC and restart them after the `FLASHBACK` is completed. Otherwise, replication tasks might fail. - For more information, see [User document](/sql-statements/sql-statement-flashback-to-timestamp.md). + For more information, see [User document](/sql-statements/sql-statement-flashback-cluster.md). * Support restoring a deleted database by using `FLASHBACK DATABASE` [#20463](https://github.com/pingcap/tidb/issues/20463) @[erwadba](https://github.com/erwadba) @@ -168,7 +169,7 @@ In v6.4.0-DMR, the key new features and improvements are as follows: * Be compatible with the Linear Hash partitioning syntax [#38450](https://github.com/pingcap/tidb/issues/38450) @[mjonss](https://github.com/mjonss) - In the earlier version, TiDB has supported the Hash, Range, and List partitioning. Starting from v6.4.0, TiDB can also be compatible with the syntaxt of [MySQL Linear Hash partitioning](https://dev.mysql.com/doc/refman/5.7/en/partitioning-linear-hash.html). + In the earlier version, TiDB has supported the Hash, Range, and List partitioning. Starting from v6.4.0, TiDB can also be compatible with the syntax of [MySQL Linear Hash partitioning](https://dev.mysql.com/doc/refman/5.7/en/partitioning-linear-hash.html). In TiDB, you can execute the existing DDL statements of your MySQL Linear Hash partitions directly, and TiDB will create the corresponding Hash partition tables (note that there is no Linear Hash partition inside TiDB). You can also execute the existing DML statements of your MySQL Linear Hash partitions directly, and TiDB will return the query result of the corresponding TiDB Hash partitions normally. This feature ensures the TiDB syntax compatibility with MySQL Linear Hash partitions and facilitates seamless migration from MySQL-based applications to TiDB. @@ -277,8 +278,9 @@ In v6.4.0-DMR, the key new features and improvements are as follows: | Variable name | Change type | Description | |--------|------------------------------|------| +| [`max_execution_time`](/system-variables.md#max_execution_time) | Modified | Before v6.4.0, this variable takes effect on all types of statements. Starting from v6.4.0, this variable only controls the maximum execution time of read-only statements. | | [`tidb_constraint_check_in_place_pessimistic`](/system-variables.md#tidb_constraint_check_in_place_pessimistic-new-in-v630) | Modified | Removes the GLOBAL scope and allows you to modify the default value using the [`pessimistic-txn.constraint-check-in-place-pessimistic`](/tidb-configuration-file.md#constraint-check-in-place-pessimistic-new-in-v640) configuration item. This variable controls when TiDB checks the unique constraints in pessimistic transactions. | -| [`tidb_ddl_flashback_concurrency`](/system-variables.md#tidb_ddl_flashback_concurrency-new-in-v630) | Modified | Takes effect starting from v6.4.0 and controls the concurrency of [`FLASHBACK CLUSTER TO TIMESTAMP`](/sql-statements/sql-statement-flashback-to-timestamp.md). The default value is `64`. | +| [`tidb_ddl_flashback_concurrency`](/system-variables.md#tidb_ddl_flashback_concurrency-new-in-v630) | Modified | Takes effect starting from v6.4.0 and controls the concurrency of [`FLASHBACK CLUSTER TO TIMESTAMP`](/sql-statements/sql-statement-flashback-cluster.md). The default value is `64`. | | [`tidb_enable_clustered_index`](/system-variables.md#tidb_enable_clustered_index-new-in-v50) | Modified | Changes the default value from `INT_ONLY` to `ON`, meaning that primary keys are created as clustered indexes by default.| | [`tidb_enable_paging`](/system-variables.md#tidb_enable_paging-new-in-v540) | Modified | Changes the default value from `OFF` to `ON`, meaning that the method of paging to send coprocessor requests is used by default. | | [`tidb_enable_prepared_plan_cache`](/system-variables.md#tidb_enable_prepared_plan_cache-new-in-v610) | Modified | Adds the SESSION scope. This variable controls whether to enable [Prepared Plan Cache](/sql-prepared-plan-cache.md). | @@ -340,7 +342,7 @@ In v6.4.0-DMR, the key new features and improvements are as follows: - Add a new configuration item `apply-yield-write-size` to control the maximum number of bytes that the Apply thread can write for one Finite-state Machine in one round of poll, and relieve Raftstore congestion when the Apply thread writes a large volume of data [#13313](https://github.com/tikv/tikv/issues/13313) @[glorv](https://github.com/glorv) - Warm up the entry cache before migrating the leader of Region to avoid QPS jitter during the leader transfer process [#13060](https://github.com/tikv/tikv/issues/13060) @[cosven](https://github.com/cosven) - - Support pushing down the `json_constains` operator to Coprocessor [#13592](https://github.com/tikv/tikv/issues/13592) @[lizhenhuan](https://github.com/lizhenhuan) + - Support pushing down the `json_constrains` operator to Coprocessor [#13592](https://github.com/tikv/tikv/issues/13592) @[lizhenhuan](https://github.com/lizhenhuan) - Add the asynchronous function for `CausalTsProvider` to improve the flush performance in some scenarios [#13428](https://github.com/tikv/tikv/issues/13428) @[zeminzhou](https://github.com/zeminzhou) + PD @@ -371,7 +373,7 @@ In v6.4.0-DMR, the key new features and improvements are as follows: + TiCDC - Support replicating the exchange partition DDL statements [#639](https://github.com/pingcap/tiflow/issues/639) @[asddongmen](https://github.com/asddongmen) - - Improve non-batch sending performance for the MQ sink module [#7353](https://github.com/pingcap/tiflow/issues/7353) @[hi-rustin](https://github.com/hi-rustin) + - Improve non-batch sending performance for the MQ sink module [#7353](https://github.com/pingcap/tiflow/issues/7353) @[hi-rustin](https://github.com/Rustin170506) - Improve performance of TiCDC puller when a table has a large number of Regions [#7078](https://github.com/pingcap/tiflow/issues/7078) [#7281](https://github.com/pingcap/tiflow/issues/7281) @[sdojjy](https://github.com/sdojjy) - Support reading historical data in the downstream TiDB by using the `tidb_enable_external_ts_read` variable when Syncpoint is enabled [#7419](https://github.com/pingcap/tiflow/issues/7419) @[asddongmen](https://github.com/asddongmen) - Enable transaction split and disable safeMode by default to improve the replication stability [#7505](https://github.com/pingcap/tiflow/issues/7505) @[asddongmen](https://github.com/asddongmen) @@ -436,9 +438,9 @@ In v6.4.0-DMR, the key new features and improvements are as follows: - Fix the issue that `sasl-password` in the `changefeed query` result is not masked [#7182](https://github.com/pingcap/tiflow/issues/7182) @[dveeden](https://github.com/dveeden) - Fix the issue that TiCDC might become unavailable when too many operations in an etcd transaction are committed [#7131](https://github.com/pingcap/tiflow/issues/7131) @[asddongmen](https://github.com/asddongmen) - Fix the issue that redo logs might be deleted incorrectly [#6413](https://github.com/pingcap/tiflow/issues/6413) @[asddongmen](https://github.com/asddongmen) - - Fix performance regression when replicating wide tables in Kafka Sink V2 [#7344](https://github.com/pingcap/tiflow/issues/7344) @[hi-rustin](https://github.com/hi-rustin) - - Fix the issue that checkpoint ts might be advanced incorrectly [#7274](https://github.com/pingcap/tiflow/issues/7274) @[hi-rustin](https://github.com/hi-rustin) - - Fix the issue that too many logs are printed due to improper log level of the mounter module [#7235](https://github.com/pingcap/tiflow/issues/7235) @[hi-rustin](https://github.com/hi-rustin) + - Fix performance regression when replicating wide tables in Kafka Sink V2 [#7344](https://github.com/pingcap/tiflow/issues/7344) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that checkpoint ts might be advanced incorrectly [#7274](https://github.com/pingcap/tiflow/issues/7274) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that too many logs are printed due to improper log level of the mounter module [#7235](https://github.com/pingcap/tiflow/issues/7235) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that a TiCDC cluster might have two owners [#4051](https://github.com/pingcap/tiflow/issues/4051) @[asddongmen](https://github.com/asddongmen) + TiDB Data Migration (DM) diff --git a/releases/release-6.5.0.md b/releases/release-6.5.0.md index f08b24fb32795..9b5967a17b53f 100644 --- a/releases/release-6.5.0.md +++ b/releases/release-6.5.0.md @@ -9,7 +9,7 @@ Release date: December 29, 2022 TiDB version: 6.5.0 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.5.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) TiDB 6.5.0 is a Long-Term Support Release (LTS). @@ -25,7 +25,7 @@ Compared with TiDB [6.4.0-DMR](/releases/release-6.4.0.md), TiDB 6.5.0 introduce - The [index acceleration](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) feature becomes generally available (GA), which improves the performance of adding indexes by about 10 times compared with v6.1.0. - The TiDB global memory control becomes GA, and you can control the memory consumption threshold via [`tidb_server_memory_limit`](/system-variables.md#tidb_server_memory_limit-new-in-v640). - The high-performance and globally monotonic [`AUTO_INCREMENT`](/auto-increment.md#mysql-compatibility-mode) column attribute becomes GA, which is compatible with MySQL. -- [`FLASHBACK CLUSTER TO TIMESTAMP`](/sql-statements/sql-statement-flashback-to-timestamp.md) is now compatible with TiCDC and PITR and becomes GA. +- [`FLASHBACK CLUSTER TO TIMESTAMP`](/sql-statements/sql-statement-flashback-cluster.md) is now compatible with TiCDC and PITR and becomes GA. - Enhance TiDB optimizer by making the more accurate [Cost Model version 2](/cost-model.md#cost-model-version-2) generally available and supporting expressions connected by `AND` for [INDEX MERGE](/explain-index-merge.md). - Support pushing down the `JSON_EXTRACT()` function to TiFlash. - Support [password management](/password-management.md) policies that meet password compliance auditing requirements. @@ -52,9 +52,9 @@ Compared with TiDB [6.4.0-DMR](/releases/release-6.4.0.md), TiDB 6.5.0 introduce * Support restoring a cluster to a specific point in time by using `FLASHBACK CLUSTER TO TIMESTAMP` (GA) [#37197](https://github.com/pingcap/tidb/issues/37197) [#13303](https://github.com/tikv/tikv/issues/13303) @[Defined2014](https://github.com/Defined2014) @[bb7133](https://github.com/bb7133) @[JmPotato](https://github.com/JmPotato) @[Connor1996](https://github.com/Connor1996) @[HuSharp](https://github.com/HuSharp) @[CalvinNeo](https://github.com/CalvinNeo) - Since v6.4.0, TiDB has introduced the [`FLASHBACK CLUSTER TO TIMESTAMP`](/sql-statements/sql-statement-flashback-to-timestamp.md) statement as an experimental feature. You can use this statement to restore a cluster to a specific point in time within the Garbage Collection (GC) life time. In v6.5.0, this feature is now compatible with TiCDC and PITR and becomes GA. This feature helps you to easily undo DML misoperations, restore the original cluster in minutes, and roll back data at different time points to determine the exact time when data changes. + Since v6.4.0, TiDB has introduced the [`FLASHBACK CLUSTER TO TIMESTAMP`](/sql-statements/sql-statement-flashback-cluster.md) statement as an experimental feature. You can use this statement to restore a cluster to a specific point in time within the Garbage Collection (GC) life time. In v6.5.0, this feature is now compatible with TiCDC and PITR and becomes GA. This feature helps you to easily undo DML misoperations, restore the original cluster in minutes, and roll back data at different time points to determine the exact time when data changes. - For more information, see [documentation](/sql-statements/sql-statement-flashback-to-timestamp.md). + For more information, see [documentation](/sql-statements/sql-statement-flashback-cluster.md). * Fully support non-transactional DML statements including `INSERT`, `REPLACE`, `UPDATE`, and `DELETE` [#33485](https://github.com/pingcap/tidb/issues/33485) @[ekexium](https://github.com/ekexium) @@ -132,7 +132,7 @@ Compared with TiDB [6.4.0-DMR](/releases/release-6.4.0.md), TiDB 6.5.0 introduce - The user can still access TiDB Dashboard for diagnosis even if the PD node is unavailable. - Accessing TiDB Dashboard on the internet does not involve the privileged interfaces of PD. Therefore, the security risk of the cluster is reduced. - For more information, see [documentation](https://docs.pingcap.com/tidb-in-kubernetes/dev/get-started#deploy-tidb-dashboard-independently). + For more information, see [documentation](https://docs.pingcap.com/tidb-in-kubernetes/v1.5/get-started#deploy-tidb-dashboard-independently). * Performance Overview dashboard adds TiFlash and CDC (Change Data Capture) panels [#39230](https://github.com/pingcap/tidb/issues/39230) @[dbsid](https://github.com/dbsid) @@ -319,10 +319,12 @@ Compared with TiDB [6.4.0-DMR](/releases/release-6.4.0.md), TiDB 6.5.0 introduce | [`tidb_server_memory_limit`](/system-variables.md#tidb_server_memory_limit-new-in-v640) | Modified | Changes the default value from `0` to `80%`. As the TiDB global memory control becomes GA, this default value change enables the memory control by default and sets the memory limit for a TiDB instance to 80% of the total memory by default. | | [`default_password_lifetime`](/system-variables.md#default_password_lifetime-new-in-v650) | Newly added | Sets the global policy for automatic password expiration to require users to change passwords periodically. The default value `0` indicates that passwords never expire. | | [`disconnect_on_expired_password`](/system-variables.md#disconnect_on_expired_password-new-in-v650) | Newly added | Indicates whether TiDB disconnects the client connection when the password is expired. This variable is read-only. | +| [`tidb_store_batch_size`](/system-variables.md#tidb_store_batch_size) | Newly added | This variable is disabled by default, and the feature controlled by it is not stable yet. It is not recommended to modify this variable in production environments. | | [`password_history`](/system-variables.md#password_history-new-in-v650) | Newly added | This variable is used to establish a password reuse policy that allows TiDB to limit password reuse based on the number of password changes. The default value `0` means disabling the password reuse policy based on the number of password changes. | | [`password_reuse_interval`](/system-variables.md#password_reuse_interval-new-in-v650) | Newly added | This variable is used to establish a password reuse policy that allows TiDB to limit password reuse based on time elapsed. The default value `0` means disabling the password reuse policy based on time elapsed. | | [`tidb_auto_build_stats_concurrency`](/system-variables.md#tidb_auto_build_stats_concurrency-new-in-v650) | Newly added | This variable is used to set the concurrency of executing the automatic update of statistics. The default value is `1`. | | [`tidb_cdc_write_source`](/system-variables.md#tidb_cdc_write_source-new-in-v650) | Newly added | When this variable is set to a value other than 0, data written in this session is considered to be written by TiCDC. This variable can only be modified by TiCDC. Do not manually modify this variable in any case. | +| [`tidb_enable_plan_replayer_capture`](/system-variables.md#tidb_enable_plan_replayer_capture) | Newly added | The feature controlled by this variable is not fully functional in TiDB v6.5.0. Do not change the default value. | | [`tidb_index_merge_intersection_concurrency`](/system-variables.md#tidb_index_merge_intersection_concurrency-new-in-v650) | Newly added | Sets the maximum concurrency for the intersection operations that index merge performs. It is effective only when TiDB accesses partitioned tables in the dynamic pruning mode. | | [`tidb_source_id`](/system-variables.md#tidb_source_id-new-in-v650) | Newly added | This variable is used to configure the different cluster IDs in a [bi-directional replication](/ticdc/ticdc-bidirectional-replication.md) cluster.| | [`tidb_sysproc_scan_concurrency`](/system-variables.md#tidb_sysproc_scan_concurrency-new-in-v650) | Newly added | This variable is used to set the concurrency of scan operations performed when TiDB executes internal SQL statements (such as an automatic update of statistics). The default value is `1`. | @@ -351,7 +353,8 @@ Compared with TiDB [6.4.0-DMR](/releases/release-6.4.0.md), TiDB 6.5.0 introduce | TiDB | [`server-memory-quota`](/tidb-configuration-file.md#server-memory-quota-new-in-v409) | Deprecated | Starting from v6.5.0, this configuration item is deprecated. Instead, use the system variable [`tidb_server_memory_limit`](/system-variables.md#tidb_server_memory_limit-new-in-v640) to manage memory globally. | | TiDB | [`disconnect-on-expired-password`](/tidb-configuration-file.md#disconnect-on-expired-password-new-in-v650) | Newly added | Determines whether TiDB disconnects the client connection when the password is expired. The default value is `true`, which means the client connection is disconnected when the password is expired. | | TiKV | `raw-min-ts-outlier-threshold` | Deleted | This configuration item was deprecated in v6.4.0 and deleted in v6.5.0. | -| TiKV | [`cdc.min-ts-interval`](/tikv-configuration-file.md#min-ts-interval) | Modified | To reduce CDC latency, the default value is changed from `1s` to `200ms`. | +| TiKV | [`raft-engine.bytes-per-sync`](/tikv-configuration-file.md#bytes-per-sync-2) | Deprecated | Starting from v6.5.0, Raft Engine writes logs to disk directly without buffering. Therefore, this configuration item is deprecated and no longer functional. | +| TiKV | [`cdc.min-ts-interval`](/tikv-configuration-file.md#min-ts-interval) | Modified | To reduce CDC latency, the default value is changed from `"1s"` to `"200ms"`. | | TiKV | [`memory-use-ratio`](/tikv-configuration-file.md#memory-use-ratio-new-in-v650) | Newly added | Indicates the ratio of available memory to total system memory in PITR log recovery. | | TiCDC | [`sink.terminator`](/ticdc/ticdc-changefeed-config.md#changefeed-configuration-parameters) | Newly added | Indicates the row terminator, which is used for separating two data change events. The value is empty by default, which means `\r\n` is used. | | TiCDC | [`sink.date-separator`](/ticdc/ticdc-changefeed-config.md#changefeed-configuration-parameters) | Newly added | Indicates the date separator type of the file directory. Value options are `none`, `year`, `month`, and `day`. `none` is the default value and means that the date is not separated. | diff --git a/releases/release-6.5.1.md b/releases/release-6.5.1.md index d2be099af32e5..e0768b24ac2a2 100644 --- a/releases/release-6.5.1.md +++ b/releases/release-6.5.1.md @@ -9,7 +9,7 @@ Release date: March 10, 2023 TiDB version: 6.5.1 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.5.1#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) ## Compatibility changes @@ -25,6 +25,8 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - The default value of the TiKV [`advance-ts-interval`](/tikv-configuration-file.md#advance-ts-interval) configuration item is changed from `1s` to `20s`. You can modify this configuration item to reduce the latency and improve the timeliness of the Stale Read data. See [Reduce Stale Read latency](/stale-read.md#reduce-stale-read-latency) for details. +- The default value of the TiKV [`cdc.min-ts-interval`](/tikv-configuration-file.md#min-ts-interval) configuration item is changed from `"200ms"` to `"1s"` to reduce network traffic. + ## Improvements + TiDB @@ -50,7 +52,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Support starting TiKV on a CPU with less than 1 core [#13586](https://github.com/tikv/tikv/issues/13586) [#13752](https://github.com/tikv/tikv/issues/13752) [#14017](https://github.com/tikv/tikv/issues/14017) @[andreid-db](https://github.com/andreid-db) - Increase the thread limit of the Unified Read Pool (`readpool.unified.max-thread-count`) to 10 times the CPU quota, to better handle high-concurrency queries [#13690](https://github.com/tikv/tikv/issues/13690) @[v01dstar](https://github.com/v01dstar) - - Change the the default value of `resolved-ts.advance-ts-interval` from `"1s"` to `"20s"`, to reduce cross-region traffic [#14100](https://github.com/tikv/tikv/issues/14100) @[overvenus](https://github.com/overvenus) + - Change the default value of `resolved-ts.advance-ts-interval` from `"1s"` to `"20s"`, to reduce cross-region traffic [#14100](https://github.com/tikv/tikv/issues/14100) @[overvenus](https://github.com/overvenus) + TiFlash @@ -64,7 +66,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- + TiCDC - - Enable pull-based sink to optimize system throughput [#8232](https://github.com/pingcap/tiflow/issues/8232) @[hi-rustin](https://github.com/hi-rustin) + - Enable pull-based sink to optimize system throughput [#8232](https://github.com/pingcap/tiflow/issues/8232) @[hi-rustin](https://github.com/Rustin170506) - Support storing redo logs to GCS-compatible or Azure-compatible object storage [#7987](https://github.com/pingcap/tiflow/issues/7987) @[CharlesCheung96](https://github.com/CharlesCheung96) - Implement MQ sink and MySQL sink in the asynchronous mode to improve the sink throughput [#5928](https://github.com/pingcap/tiflow/issues/5928) @[amyangfei](https://github.com/amyangfei) @[CharlesCheung96](https://github.com/CharlesCheung96) @@ -163,11 +165,11 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Fix the issue that changefeed might get stuck in special scenarios such as when scaling in or scaling out TiKV or TiCDC nodes [#8174](https://github.com/pingcap/tiflow/issues/8174) @[hicqu](https://github.com/hicqu) - Fix the issue that precheck is not performed on the storage path of redo log [#6335](https://github.com/pingcap/tiflow/issues/6335) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the issue of insufficient duration that redo log can tolerate for S3 storage failure [#8089](https://github.com/pingcap/tiflow/issues/8089) @[CharlesCheung96](https://github.com/CharlesCheung96) - - Fix the issue that `transaction_atomicity` and `protocol` cannot be updated via the configuration file [#7935](https://github.com/pingcap/tiflow/issues/7935) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that `transaction-atomicity` and `protocol` cannot be updated via the configuration file [#7935](https://github.com/pingcap/tiflow/issues/7935) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the issue that the checkpoint cannot advance when TiCDC replicates an excessively large number of tables [#8004](https://github.com/pingcap/tiflow/issues/8004) @[overvenus](https://github.com/overvenus) - Fix the issue that applying redo log might cause OOM when the replication lag is excessively high [#8085](https://github.com/pingcap/tiflow/issues/8085) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the issue that the performance degrades when redo log is enabled to write meta [#8074](https://github.com/pingcap/tiflow/issues/8074) @[CharlesCheung96](https://github.com/CharlesCheung96) - - Fix a bug that the context deadline is exceeded when TiCDC replicates data without splitting large transactions [#7982](https://github.com/pingcap/tiflow/issues/7982) @[hi-rustin](https://github.com/hi-rustin) + - Fix a bug that the context deadline is exceeded when TiCDC replicates data without splitting large transactions [#7982](https://github.com/pingcap/tiflow/issues/7982) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that pausing a changefeed when PD is abnormal results in incorrect status [#8330](https://github.com/pingcap/tiflow/issues/8330) @[sdojjy](https://github.com/sdojjy) - Fix the data inconsistency that occurs when replicating data to a TiDB or MySQL sink and when `CHARACTER SET` is specified on the column that has the non-null unique index without a primary key [#8420](https://github.com/pingcap/tiflow/issues/8420) @[asddongmen](https://github.com/asddongmen) - Fix the panic issue in table scheduling and blackhole sink [#8024](https://github.com/pingcap/tiflow/issues/8024) [#8142](https://github.com/pingcap/tiflow/issues/8142) @[hicqu](https://github.com/hicqu) @@ -182,6 +184,6 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Fix the issue that TiDB Lightning prechecks cannot find dirty data left by previously failed imports [#39477](https://github.com/pingcap/tidb/issues/39477) @[dsdashun](https://github.com/dsdashun) - Fix the issue that TiDB Lightning panics in the split-region phase [#40934](https://github.com/pingcap/tidb/issues/40934) @[lance6716](https://github.com/lance6716) - - Fix the issue that the conflict resolution logic (`duplicate-resolution`) might lead to inconsistent checksums [#40657](https://github.com/pingcap/tidb/issues/40657) @[gozssky](https://github.com/gozssky) + - Fix the issue that the conflict resolution logic (`duplicate-resolution`) might lead to inconsistent checksums [#40657](https://github.com/pingcap/tidb/issues/40657) @[sleepymole](https://github.com/sleepymole) - Fix the issue that TiDB Lightning might incorrectly skip conflict resolution when all but the last TiDB Lightning instance encounters a local duplicate record during a parallel import [#40923](https://github.com/pingcap/tidb/issues/40923) @[lichunzhu](https://github.com/lichunzhu) - Fix the issue that when importing data in Local Backend mode, the target columns do not automatically generate data if the compound primary key of the imported target table has an `auto_random` column and no value for the column is specified in the source data [#41454](https://github.com/pingcap/tidb/issues/41454) @[D3Hunter](https://github.com/D3Hunter) diff --git a/releases/release-6.5.10.md b/releases/release-6.5.10.md new file mode 100644 index 0000000000000..9971b6c14ab4d --- /dev/null +++ b/releases/release-6.5.10.md @@ -0,0 +1,148 @@ +--- +title: TiDB 6.5.10 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 6.5.10. +--- + +# TiDB 6.5.10 Release Notes + +Release date: June 20, 2024 + +TiDB version: 6.5.10 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) + +## Compatibility changes + +- In earlier versions, when processing a transaction containing `UPDATE` changes, if the primary key or non-null unique index value is modified in an `UPDATE` event, TiCDC splits this event into `DELETE` and `INSERT` events. Starting from v6.5.10, when using the MySQL sink, TiCDC splits an `UPDATE` event into `DELETE` and `INSERT` events if the transaction `commitTS` for the `UPDATE` change is less than TiCDC `thresholdTS` (which is the current timestamp fetched from PD when TiCDC starts replicating the corresponding table to the downstream). This behavior change addresses the issue of downstream data inconsistencies caused by the potentially incorrect order of `UPDATE` events received by TiCDC, which can lead to an incorrect order of split `DELETE` and `INSERT` events. For more information, see [documentation](https://docs.pingcap.com/tidb/v6.5/ticdc-split-update-behavior#split-update-events-for-mysql-sinks). [#10918](https://github.com/pingcap/tiflow/issues/10918) @[lidezhu](https://github.com/lidezhu) +- Must set the line terminator when using TiDB Lightning `strict-format` to import CSV files [#37338](https://github.com/pingcap/tidb/issues/37338) @[lance6716](https://github.com/lance6716) + +## Improvements + ++ TiDB + + - Improve the MySQL compatibility of expression default values displayed in the output of `SHOW CREATE TABLE` [#52939](https://github.com/pingcap/tidb/issues/52939) @[CbcWestwolf](https://github.com/CbcWestwolf) + - Remove stores without Regions during MPP load balancing [#52313](https://github.com/pingcap/tidb/issues/52313) @[xzhangxian1008](https://github.com/xzhangxian1008) + ++ TiKV + + - Accelerate the shutdown speed of TiKV [#16680](https://github.com/tikv/tikv/issues/16680) @[LykxSassinator](https://github.com/LykxSassinator) + - Add monitoring metrics for the queue time for processing CDC events to facilitate troubleshooting downstream CDC event latency issues [#16282](https://github.com/tikv/tikv/issues/16282) @[hicqu](https://github.com/hicqu) + ++ Tools + + + Backup & Restore (BR) + + - BR cleans up empty SST files during data recovery [#16005](https://github.com/tikv/tikv/issues/16005) @[Leavrth](https://github.com/Leavrth) + - Increase the number of retries for failures caused by DNS errors [#53029](https://github.com/pingcap/tidb/issues/53029) @[YuJuncen](https://github.com/YuJuncen) + - Increase the number of retries for failures caused by the absence of a leader in a Region [#54017](https://github.com/pingcap/tidb/issues/54017) @[Leavrth](https://github.com/Leavrth) + + + TiCDC + + - Support directly outputting raw events when the downstream is a Message Queue (MQ) or cloud storage [#11211](https://github.com/pingcap/tiflow/issues/11211) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Improve memory stability during data recovery using redo logs to reduce the probability of OOM [#10900](https://github.com/pingcap/tiflow/issues/10900) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Significantly improve the stability of data replication in transaction conflict scenarios, with up to 10 times performance improvement [#10896](https://github.com/pingcap/tiflow/issues/10896) @[CharlesCheung96](https://github.com/CharlesCheung96) + +## Bug fixes + ++ TiDB + + - Fix the issue that querying metadata during statistics initialization might cause OOM [#52219](https://github.com/pingcap/tidb/issues/52219) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that for tables containing auto-increment columns with `AUTO_ID_CACHE=1`, setting `auto_increment_increment` and `auto_increment_offset` system variables to non-default values might cause incorrect auto-increment ID allocation [#52622](https://github.com/pingcap/tidb/issues/52622) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that restoring a table with `AUTO_ID_CACHE=1` using the `RESTORE` statement might cause a `Duplicate entry` error [#52680](https://github.com/pingcap/tidb/issues/52680) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the `STATE` field in the `INFORMATION_SCHEMA.TIDB_TRX` table is empty due to the `size` of the `STATE` field not being defined [#53026](https://github.com/pingcap/tidb/issues/53026) @[cfzjywxk](https://github.com/cfzjywxk) + - Fix the issue that TiDB does not create corresponding statistics metadata (`stats_meta`) when creating a table with foreign keys [#53652](https://github.com/pingcap/tidb/issues/53652) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the statistics synchronous loading mechanism might fail unexpectedly under high query concurrency [#52294](https://github.com/pingcap/tidb/issues/52294) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `Distinct_count` information in GlobalStats might be incorrect [#53752](https://github.com/pingcap/tidb/issues/53752) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the histogram and TopN in the primary key column statistics are not loaded after restarting TiDB [#37548](https://github.com/pingcap/tidb/issues/37548) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that certain filter conditions in queries might cause the planner module to report an `invalid memory address or nil pointer dereference` error [#53582](https://github.com/pingcap/tidb/issues/53582) [#53580](https://github.com/pingcap/tidb/issues/53580) [#53594](https://github.com/pingcap/tidb/issues/53594) [#53603](https://github.com/pingcap/tidb/issues/53603) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that `PREPARE`/`EXECUTE` statements with the `CONV` expression containing a `?` argument might result in incorrect query results when executed multiple times [#53505](https://github.com/pingcap/tidb/issues/53505) @[qw4990](https://github.com/qw4990) + - Fix the issue of incorrect WARNINGS information when using Optimizer Hints [#53767](https://github.com/pingcap/tidb/issues/53767) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the query latency of stale reads increases, caused by information schema cache misses [#53428](https://github.com/pingcap/tidb/issues/53428) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that DDL statements incorrectly use etcd and cause tasks to queue up [#52335](https://github.com/pingcap/tidb/issues/52335) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that internal columns are not renamed when executing `RENAME INDEX` to rename expression indexes [#51431](https://github.com/pingcap/tidb/issues/51431) @[ywqzzy](https://github.com/ywqzzy) + - Fix the issue that executing `CREATE OR REPLACE VIEW` concurrently might result in the `table doesn't exist` error [#53673](https://github.com/pingcap/tidb/issues/53673) @[tangenta](https://github.com/tangenta) + - Fix the issue that TiDB might panic when the JOIN condition contains an implicit type conversion [#46556](https://github.com/pingcap/tidb/issues/46556) @[qw4990](https://github.com/qw4990) + - Fix the issue that DDL operations get stuck due to network problems [#47060](https://github.com/pingcap/tidb/issues/47060) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that IndexJoin produces duplicate rows when calculating hash values in the Left Outer Anti Semi type [#52902](https://github.com/pingcap/tidb/issues/52902) @[yibin87](https://github.com/yibin87) + - Fix the issue that subqueries included in the `ALL` function might cause incorrect results [#52755](https://github.com/pingcap/tidb/issues/52755) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `TIMESTAMPADD()` function returns incorrect results [#41052](https://github.com/pingcap/tidb/issues/41052) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that TiDB might crash when `tidb_mem_quota_analyze` is enabled and the memory used by updating statistics exceeds the limit [#52601](https://github.com/pingcap/tidb/issues/52601) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that subqueries in an `UPDATE` list might cause TiDB to panic [#52687](https://github.com/pingcap/tidb/issues/52687) @[winoros](https://github.com/winoros) + - Fix the overflow issue of the `Longlong` type in predicates [#45783](https://github.com/pingcap/tidb/issues/45783) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue of inconsistent data indexes caused by concurrent DML operations when adding a unique index [#52914](https://github.com/pingcap/tidb/issues/52914) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that TiDB might panic when parsing index data [#47115](https://github.com/pingcap/tidb/issues/47115) @[zyguan](https://github.com/zyguan) + - Fix the issue that column pruning without using shallow copies of slices might cause TiDB to panic [#52768](https://github.com/pingcap/tidb/issues/52768) @[winoros](https://github.com/winoros) + - Fix the issue that using a view does not work in recursive CTE [#49721](https://github.com/pingcap/tidb/issues/49721) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `LEADING` hint does not support querying block aliases [#44645](https://github.com/pingcap/tidb/issues/44645) @[qw4990](https://github.com/qw4990) + - Fix the incorrect result of the TopN operator in correlated subqueries [#52777](https://github.com/pingcap/tidb/issues/52777) @[yibin87](https://github.com/yibin87) + - Fix the issue that unstable unique IDs of columns might cause the `UPDATE` statement to return errors [#53236](https://github.com/pingcap/tidb/issues/53236) @[winoros](https://github.com/winoros) + - Fix the issue that TiDB keeps sending probe requests to a TiFlash node that has been offline [#46602](https://github.com/pingcap/tidb/issues/46602) @[zyguan](https://github.com/zyguan) + - Fix the issue that comparing a column of `YEAR` type with an unsigned integer that is out of range causes incorrect results [#50235](https://github.com/pingcap/tidb/issues/50235) @[qw4990](https://github.com/qw4990) + - Fix the issue that AutoID Leader change might cause the value of the auto-increment column to decrease in the case of `AUTO_ID_CACHE=1` [#52600](https://github.com/pingcap/tidb/issues/52600) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that non-BIGINT unsigned integers might produce incorrect results when compared with strings/decimals [#41736](https://github.com/pingcap/tidb/issues/41736) @[LittleFall](https://github.com/LittleFall) + - Fix the issue that data conversion from the `FLOAT` type to the `UNSIGNED` type returns incorrect results [#41736](https://github.com/pingcap/tidb/issues/41736) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that `VAR_SAMP()` cannot be used as a window function [#52933](https://github.com/pingcap/tidb/issues/52933) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that a wrong TableDual plan causes empty query results [#50051](https://github.com/pingcap/tidb/issues/50051) @[onlyacat](https://github.com/onlyacat) + - Fix the issue that the TiDB synchronously loading statistics mechanism retries to load empty statistics indefinitely and prints the `fail to get stats version for this histogram` log [#52657](https://github.com/pingcap/tidb/issues/52657) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that an empty projection causes TiDB to panic [#49109](https://github.com/pingcap/tidb/issues/49109) @[winoros](https://github.com/winoros) + - Fix the issue that the TopN operator might be pushed down incorrectly [#37986](https://github.com/pingcap/tidb/issues/37986) @[qw4990](https://github.com/qw4990) + - Fix the issue that TiDB panics when executing the `SHOW ERRORS` statement with a predicate that is always `true` [#46962](https://github.com/pingcap/tidb/issues/46962) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that the metadata lock fails to prevent DDL operations from executing in the plan cache scenario [#51407](https://github.com/pingcap/tidb/issues/51407) @[wjhuang2016](https://github.com/wjhuang2016) + ++ TiKV + + - Fix the issue that slow `check-leader` operations on one TiKV node cause `resolved-ts` on other TiKV nodes to fail to advance normally [#15999](https://github.com/tikv/tikv/issues/15999) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that the `CONV()` function in queries might overflow during numeric system conversion, leading to TiKV panic [#16969](https://github.com/tikv/tikv/issues/16969) @[gengliqi](https://github.com/gengliqi) + - Fix the issue of unstable test cases, ensuring that each test uses an independent temporary directory to avoid online configuration changes affecting other test cases [#16871](https://github.com/tikv/tikv/issues/16871) @[glorv](https://github.com/glorv) + - Fix the issue that the decimal part of the `DECIMAL` type is incorrect in some cases [#16913](https://github.com/tikv/tikv/issues/16913) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that resolve-ts is blocked when a stale Region peer ignores the GC message [#16504](https://github.com/tikv/tikv/issues/16504) @[crazycs520](https://github.com/crazycs520) + ++ PD + + - Fix the issue that the Leader fails to transfer when you switch it between two deployed data centers [#7992](https://github.com/tikv/pd/issues/7992) @[TonsnakeLin](https://github.com/TonsnakeLin) + - Fix the issue that down peers might not recover when using Placement Rules [#7808](https://github.com/tikv/pd/issues/7808) @[rleungx](https://github.com/rleungx) + - Fix the issue that the `Filter target` monitoring metric for PD does not provide scatter range information [#8125](https://github.com/tikv/pd/issues/8125) @[HuSharp](https://github.com/HuSharp) + ++ TiFlash + + - Fix the issue that TiFlash might fail to synchronize schemas after executing `ALTER TABLE ... EXCHANGE PARTITION` across databases [#7296](https://github.com/pingcap/tiflash/issues/7296) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that a query with an empty key range fails to correctly generate read tasks on TiFlash, which might block TiFlash queries [#9108](https://github.com/pingcap/tiflash/issues/9108) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that the `SUBSTRING_INDEX()` function might cause TiFlash to crash in some corner cases [#9116](https://github.com/pingcap/tiflash/issues/9116) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that TiFlash metadata might become corrupted and cause the process to panic when upgrading a cluster from a version earlier than v6.5.0 to v6.5.0 or later [#9039](https://github.com/pingcap/tiflash/issues/9039) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash might return transiently incorrect results in high-concurrency read scenarios [#8845](https://github.com/pingcap/tiflash/issues/8845) @[JinheLin](https://github.com/JinheLin) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that the test case `TestGetTSWithRetry` takes too long to execute [#52547](https://github.com/pingcap/tidb/issues/52547) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the Region fetched from PD does not have a Leader when restoring data using BR or importing data using TiDB Lightning in physical import mode [#51124](https://github.com/pingcap/tidb/issues/51124) [#50501](https://github.com/pingcap/tidb/issues/50501) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that a PD connection failure could cause the TiDB instance where the log backup advancer owner is located to panic [#52597](https://github.com/pingcap/tidb/issues/52597) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that after pausing, stopping, and rebuilding the log backup task, the task status is normal, but the checkpoint does not advance [#53047](https://github.com/pingcap/tidb/issues/53047) @[RidRisR](https://github.com/RidRisR) + - Fix the issue that data restore is slowed down due to the absence of a leader on a TiKV node [#50566](https://github.com/pingcap/tidb/issues/50566) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the global checkpoint of log backup is advanced ahead of the actual backup file write point due to TiKV restart, which might cause a small amount of backup data loss [#16809](https://github.com/tikv/tikv/issues/16809) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that the transfer of PD leaders might cause BR to panic when restoring data [#53724](https://github.com/pingcap/tidb/issues/53724) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that TiKV might panic when resuming a paused log backup task with unstable network connections to PD [#17020](https://github.com/tikv/tikv/issues/17020) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that log backup might be paused after the advancer owner migration [#53561](https://github.com/pingcap/tidb/issues/53561) @[RidRisR](https://github.com/RidRisR) + - Fix the issue that BR fails to correctly identify errors due to multiple nested retries during the restore process [#54053](https://github.com/pingcap/tidb/issues/54053) @[RidRisR](https://github.com/RidRisR) + + + TiCDC + + - Fix the issue that TiCDC fails to create a changefeed with Syncpoint enabled when the downstream database password is Base64 encoded [#10516](https://github.com/pingcap/tiflow/issues/10516) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that `DROP PRIMARY KEY` and `DROP UNIQUE KEY` statements are not replicated correctly [#10890](https://github.com/pingcap/tiflow/issues/10890) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that the default value of `TIMEZONE` type is not set according to the correct time zone [#10931](https://github.com/pingcap/tiflow/issues/10931) @[3AceShowHand](https://github.com/3AceShowHand) + + + TiDB Lightning + + - Fix the issue that killing the PD Leader causes TiDB Lightning to report the `invalid store ID 0` error during data import [#50501](https://github.com/pingcap/tidb/issues/50501) @[Leavrth](https://github.com/Leavrth) + - Fix the issue of missing data in the TiDB Lightning Grafana dashboard [#43357](https://github.com/pingcap/tidb/issues/43357) @[lichunzhu](https://github.com/lichunzhu) + - Fix the issue that TiDB Lightning might print sensitive information to logs in server mode [#36374](https://github.com/pingcap/tidb/issues/36374) @[kennytm](https://github.com/kennytm) + - Fix the issue that TiDB fails to generate auto-increment IDs and reports an error `Failed to read auto-increment value from storage engine` after importing a table with both `SHARD_ROW_ID_BITS` and `AUTO_ID_CACHE=1` set using TiDB Lightning [#52654](https://github.com/pingcap/tidb/issues/52654) @[D3Hunter](https://github.com/D3Hunter) + + + Dumpling + + - Fix the issue that Dumpling reports an error when exporting tables and views at the same time [#53682](https://github.com/pingcap/tidb/issues/53682) @[tangenta](https://github.com/tangenta) + + + TiDB Binlog + + - Fix the issue that deleting rows during the execution of `ADD COLUMN` might report an error `data and columnID count not match` when TiDB Binlog is enabled [#53133](https://github.com/pingcap/tidb/issues/53133) @[tangenta](https://github.com/tangenta) diff --git a/releases/release-6.5.11.md b/releases/release-6.5.11.md new file mode 100644 index 0000000000000..0ec56ed15edcd --- /dev/null +++ b/releases/release-6.5.11.md @@ -0,0 +1,132 @@ +--- +title: TiDB 6.5.11 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 6.5.11. +--- + +# TiDB 6.5.11 Release Notes + +Release date: September 20, 2024 + +TiDB version: 6.5.11 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) + +## Compatibility changes + +- Change the scope of the TiKV configuration item [`server.grpc-compression-type`](/tikv-configuration-file.md#grpc-compression-type): + + - In v6.5.x versions earlier than v6.5.11, this configuration item only affects the compression algorithm of gRPC messages between TiKV nodes. + - Starting from v6.5.11, this configuration item also affects the compression algorithm of gRPC response messages sent from TiKV to TiDB. Enabling compression might consume more CPU resources. [#17176](https://github.com/tikv/tikv/issues/17176) @[ekexium](https://github.com/ekexium) + +## Improvements + ++ TiDB + + - By batch deleting TiFlash placement rules, improve the processing speed of data GC after performing the `TRUNCATE` or `DROP` operation on partitioned tables [#54068](https://github.com/pingcap/tidb/issues/54068) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + ++ TiKV + + - Optimize the compaction trigger mechanism of RocksDB to accelerate disk space reclamation when handling a large number of DELETE versions [#17269](https://github.com/tikv/tikv/issues/17269) @[AndreMouche](https://github.com/AndreMouche) + ++ TiFlash + + - Reduce lock conflicts under highly concurrent data read operations and optimize short query performance [#9125](https://github.com/pingcap/tiflash/issues/9125) @[JinheLin](https://github.com/JinheLin) + - Optimize the execution efficiency of `LENGTH()` and `ASCII()` functions [#9344](https://github.com/pingcap/tiflash/issues/9344) @[xzhangxian1008](https://github.com/xzhangxian1008) + ++ Tools + + + TiCDC + + - When the downstream is TiDB with the `SUPER` permission granted, TiCDC supports querying the execution status of `ADD INDEX DDL` from the downstream database to avoid data replication failure due to timeout in retrying executing the DDL statement in some cases [#10682](https://github.com/pingcap/tiflow/issues/10682) @[CharlesCheung96](https://github.com/CharlesCheung96) + +## Bug fixes + ++ TiDB + + - Fix the issue that the recursive CTE operator incorrectly tracks memory usage [#54181](https://github.com/pingcap/tidb/issues/54181) @[guo-shaoge](https://github.com/guo-shaoge) + - Reset the parameters in the `Open` method of `PipelinedWindow` to fix the unexpected error that occurs when the `PipelinedWindow` is used as a child node of `Apply` due to the reuse of previous parameter values caused by repeated opening and closing operations [#53600](https://github.com/pingcap/tidb/issues/53600) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that the memory used by transactions might be tracked multiple times [#53984](https://github.com/pingcap/tidb/issues/53984) @[ekexium](https://github.com/ekexium) + - Fix the issue of abnormally high memory usage caused by `memTracker` not being detached when the `HashJoin` or `IndexLookUp` operator is the driven side sub-node of the `Apply` operator [#54005](https://github.com/pingcap/tidb/issues/54005) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that `INDEX_HASH_JOIN` cannot exit properly when SQL is abnormally interrupted [#54688](https://github.com/pingcap/tidb/issues/54688) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that table replication fails when the index length of the table replicated from DM exceeds the maximum length specified by `max-index-length` [#55138](https://github.com/pingcap/tidb/issues/55138) @[lance6716](https://github.com/lance6716) + - Fix the issue that indirect placeholder `?` references in a `GROUP BY` statement cannot find columns [#53872](https://github.com/pingcap/tidb/issues/53872) @[qw4990](https://github.com/qw4990) + - Fix the issue that the illegal column type `DECIMAL(0,0)` can be created in some cases [#53779](https://github.com/pingcap/tidb/issues/53779) @[tangenta](https://github.com/tangenta) + - Fix the issue that predicates cannot be pushed down properly when the filter condition of a SQL query contains virtual columns and the execution condition contains `UnionScan` [#54870](https://github.com/pingcap/tidb/issues/54870) @[qw4990](https://github.com/qw4990) + - Fix the issue that executing the `SELECT DISTINCT CAST(col AS DECIMAL), CAST(col AS SIGNED) FROM ...` query might return incorrect results [#53726](https://github.com/pingcap/tidb/issues/53726) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue of reusing wrong point get plans for `SELECT ... FOR UPDATE` [#54652](https://github.com/pingcap/tidb/issues/54652) @[qw4990](https://github.com/qw4990) + - Fix the issue that RANGE partitioned tables that are not strictly self-incrementing can be created [#54829](https://github.com/pingcap/tidb/issues/54829) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that the `TIMESTAMPADD()` function goes into an infinite loop when the first argument is `month` and the second argument is negative [#54908](https://github.com/pingcap/tidb/issues/54908) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that TiDB fails to reject unauthenticated user connections in some cases when using the `auth_socket` authentication plugin [#54031](https://github.com/pingcap/tidb/issues/54031) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that the network partition during adding indexes using the Distributed eXecution Framework (DXF) might cause inconsistent data indexes [#54897](https://github.com/pingcap/tidb/issues/54897) @[tangenta](https://github.com/tangenta) + - Fix the issue that the query might get stuck when terminated because the memory usage exceeds the limit set by `tidb_mem_quota_query` [#55042](https://github.com/pingcap/tidb/issues/55042) @[yibin87](https://github.com/yibin87) + - Fix the issue that improper use of metadata locks might lead to writing anomalous data when using the plan cache under certain circumstances [#53634](https://github.com/pingcap/tidb/issues/53634) @[zimulala](https://github.com/zimulala) + - Fix the issue that recursive CTE queries might result in invalid pointers [#54449](https://github.com/pingcap/tidb/issues/54449) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `tot_col_size` column in the `mysql.stats_histograms` table might be a negative number [#55126](https://github.com/pingcap/tidb/issues/55126) @[qw4990](https://github.com/qw4990) + - Fix the issue that obtaining the column information using `information_schema.columns` returns warning 1356 when a subquery is used as a column definition in a view definition [#54343](https://github.com/pingcap/tidb/issues/54343) @[lance6716](https://github.com/lance6716) + - Fix the issue that TiDB reports an error in the log when closing the connection in some cases [#53689](https://github.com/pingcap/tidb/issues/53689) @[jackysp](https://github.com/jackysp) + - Fix the issue that the performance of the `SELECT ... WHERE ... ORDER BY ...` statement execution is poor in some cases [#54969](https://github.com/pingcap/tidb/issues/54969) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the `SUB_PART` value in the `INFORMATION_SCHEMA.STATISTICS` table is `NULL` [#55812](https://github.com/pingcap/tidb/issues/55812) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that the query might return incorrect results instead of an error after being killed [#50089](https://github.com/pingcap/tidb/issues/50089) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that querying the `INFORMATION_SCHEMA.CLUSTER_SLOW_QUERY` table might cause TiDB to panic [#54324](https://github.com/pingcap/tidb/issues/54324) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that empty `groupOffset` in `StreamAggExec` might cause TiDB to panic [#53867](https://github.com/pingcap/tidb/issues/53867) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that disk files might not be deleted after the `Sort` operator spills and a query error occurs [#55061](https://github.com/pingcap/tidb/issues/55061) @[wshwsh12](https://github.com/wshwsh12) + - Fix the data race issue in `IndexNestedLoopHashJoin` [#49692](https://github.com/pingcap/tidb/issues/49692) @[solotzg](https://github.com/solotzg) + - Fix the issue that an error occurs when using `SHOW COLUMNS` to view columns in a view [#54964](https://github.com/pingcap/tidb/issues/54964) @[lance6716](https://github.com/lance6716) + - Fix the issue that an error occurs when a DML statement contains nested generated columns [#53967](https://github.com/pingcap/tidb/issues/53967) @[wjhuang2016](https://github.com/wjhuang2016) + ++ TiKV + + - Fix the issue that prevents master key rotation when the master key is stored in a Key Management Service (KMS) [#17410](https://github.com/tikv/tikv/issues/17410) @[hhwyt](https://github.com/hhwyt) + - Fix a traffic control issue that might occur after deleting large tables or partitions [#17304](https://github.com/tikv/tikv/issues/17304) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that TiKV might panic due to ingesting deleted `sst_importer` SST files [#15053](https://github.com/tikv/tikv/issues/15053) @[lance6716](https://github.com/lance6716) + - Fix the issue that TiKV might panic when a stale replica processes Raft snapshots, triggered by a slow split operation and immediate removal of the new replica [#17469](https://github.com/tikv/tikv/issues/17469) @[hbisheng](https://github.com/hbisheng) + - Fix the issue that TiKV might repeatedly panic when applying a corrupted Raft data snapshot [#15292](https://github.com/tikv/tikv/issues/15292) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that setting the gRPC message compression method via `grpc-compression-type` does not take effect on messages sent from TiKV to TiDB [#17176](https://github.com/tikv/tikv/issues/17176) @[ekexium](https://github.com/ekexium) + - Fix the issue that CDC and log-backup do not limit the timeout of `check_leader` using the `advance-ts-interval` configuration, causing the `resolved_ts` lag to be too large when TiKV restarts normally in some cases [#17107](https://github.com/tikv/tikv/issues/17107) @[MyonKeminta](https://github.com/MyonKeminta) + ++ PD + + - Fix the issue that some logs are not redacted [#8419](https://github.com/tikv/pd/issues/8419) @[rleungx](https://github.com/rleungx) + - Fix the issue that setting the TiKV configuration item [`coprocessor.region-split-size`](/tikv-configuration-file.md#region-split-size) to a value less than 1 MiB causes PD panic [#8323](https://github.com/tikv/pd/issues/8323) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that setting `replication.strictly-match-label` to `true` causes TiFlash to fail to start [#8480](https://github.com/tikv/pd/issues/8480) @[rleungx](https://github.com/rleungx) + - Fix the data race issue that PD encounters during operator checks [#8263](https://github.com/tikv/pd/issues/8263) @[lhy1024](https://github.com/lhy1024) + ++ TiFlash + + - Fix the issue that when using the `CAST()` function to convert a string to a datetime with a time zone or invalid characters, the result is incorrect [#8754](https://github.com/pingcap/tiflash/issues/8754) @[solotzg](https://github.com/solotzg) + - Fix the issue that TiFlash might panic when a database is deleted shortly after creation [#9266](https://github.com/pingcap/tiflash/issues/9266) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that setting the SSL certificate configuration to an empty string in TiFlash incorrectly enables TLS and causes TiFlash to fail to start [#9235](https://github.com/pingcap/tiflash/issues/9235) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that a network partition (network disconnection) between TiFlash and any PD might cause read request timeout errors [#9243](https://github.com/pingcap/tiflash/issues/9243) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that TiFlash might crash if an error occurs during the execution of a query containing outer join [#9190](https://github.com/pingcap/tiflash/issues/9190) @[windtalker](https://github.com/windtalker) + - Fix the issue that converting data types to `DECIMAL` might cause incorrect query results in some corner cases [#53892](https://github.com/pingcap/tidb/issues/53892) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that frequent `EXCHANGE PARTITION` and `DROP TABLE` operations over a long period in a cluster might slow down the replication of TiFlash table metadata and degrade the query performance [#9227](https://github.com/pingcap/tiflash/issues/9227) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that the checkpoint path of backup and restore is incompatible with some external storage [#55265](https://github.com/pingcap/tidb/issues/55265) @[Leavrth](https://github.com/Leavrth) + - Fix the inefficiency issue in scanning DDL jobs during incremental backups [#54139](https://github.com/pingcap/tidb/issues/54139) @[3pointer](https://github.com/3pointer) + - Fix the issue that the backup performance during checkpoint backups is affected due to interruptions in seeking Region leaders [#17168](https://github.com/tikv/tikv/issues/17168) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that DDLs requiring backfilling, such as `ADD INDEX` and `MODIFY COLUMN`, might not be correctly recovered during incremental restore [#54426](https://github.com/pingcap/tidb/issues/54426) @[3pointer](https://github.com/3pointer) + - Fix the issue that after a log backup PITR task fails and you stop it, the safepoints related to that task are not properly cleared in PD [#17316](https://github.com/tikv/tikv/issues/17316) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that backup tasks might get stuck if TiKV becomes unresponsive during the backup process [#53480](https://github.com/pingcap/tidb/issues/53480) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that BR logs might print sensitive credential information when log backup is enabled [#55273](https://github.com/pingcap/tidb/issues/55273) @[RidRisR](https://github.com/RidRisR) + + + TiCDC + + - Fix the issue that TiCDC might panic when the Sorter module reads disk data [#10853](https://github.com/pingcap/tiflow/issues/10853) @[hicqu](https://github.com/hicqu) + - Fix the issue that the Processor module might get stuck when the downstream Kafka is inaccessible [#11340](https://github.com/pingcap/tiflow/issues/11340) @[asddongmen](https://github.com/asddongmen) + + + TiDB Data Migration (DM) + + - Fix the issue that data replication is interrupted when the index length exceeds the default value of `max-index-length` [#11459](https://github.com/pingcap/tiflow/issues/11459) @[michaelmdeng](https://github.com/michaelmdeng) + - Fix the issue that schema tracker incorrectly handles LIST partition tables, causing DM errors [#11408](https://github.com/pingcap/tiflow/issues/11408) @[lance6716](https://github.com/lance6716) + - Fix the issue that DM returns an error when replicating the `ALTER TABLE ... DROP PARTITION` statement for LIST partitioned tables [#54760](https://github.com/pingcap/tidb/issues/54760) @[lance6716](https://github.com/lance6716) + - Fix the issue that DM does not set the default database when processing the `ALTER DATABASE` statement, which causes a replication error [#11503](https://github.com/pingcap/tiflow/issues/11503) @[lance6716](https://github.com/lance6716) + + + TiDB Lightning + + - Fix the issue that transaction conflicts occur during data import using TiDB Lightning [#49826](https://github.com/pingcap/tidb/issues/49826) @[lance6716](https://github.com/lance6716) + - Fix the issue that TiKV data might be corrupted when importing data after disabling the import mode of TiDB Lightning [#15003](https://github.com/tikv/tikv/issues/15003) [#47694](https://github.com/pingcap/tidb/issues/47694) @[lance6716](https://github.com/lance6716) + - Fix the issue that during importing data using TiDB Lightning, an error occurs when restarting TiKV [#15912](https://github.com/tikv/tikv/issues/15912) @[lance6716](https://github.com/lance6716) diff --git a/releases/release-6.5.12.md b/releases/release-6.5.12.md new file mode 100644 index 0000000000000..5b99c9c4d9147 --- /dev/null +++ b/releases/release-6.5.12.md @@ -0,0 +1,185 @@ +--- +title: TiDB 6.5.12 Release Notes +summary: Learn about the improvements and bug fixes in TiDB 6.5.12. +--- + +# TiDB 6.5.12 Release Notes + +Release date: February 27, 2025 + +TiDB version: 6.5.12 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) + +## Compatibility changes + +- Support the openEuler 22.03 LTS SP3/SP4 operating system. For more information, see [OS and platform requirements](https://docs.pingcap.com/tidb/v6.5/hardware-and-software-requirements#os-and-platform-requirements). +- Set a default limit of 2048 for DDL historical tasks retrieved through the [TiDB HTTP API](https://github.com/pingcap/tidb/blob/release-6.5/docs/tidb_http_api.md) to prevent OOM issues caused by excessive historical tasks [#55711](https://github.com/pingcap/tidb/issues/55711) @[joccau](https://github.com/joccau) +- Add a new system variable [`tidb_ddl_reorg_max_write_speed`](https://docs.pingcap.com/tidb/v6.5/system-variables#tidb_ddl_reorg_max_write_speed-new-in-v6512) to limit the maximum speed of the ingest phase when adding indexes [#57156](https://github.com/pingcap/tidb/issues/57156) @[CbcWestwolf](https://github.com/CbcWestwolf) + +## Improvements + ++ TiDB + + - Enhance the validity check for read timestamps [#57786](https://github.com/pingcap/tidb/issues/57786) @[MyonKeminta](https://github.com/MyonKeminta) + ++ TiKV + + - Add the detection mechanism for invalid `max_ts` updates [#17916](https://github.com/tikv/tikv/issues/17916) @[ekexium](https://github.com/ekexium) + ++ TiFlash + + - Improve the garbage collection speed of outdated data in the background for tables with clustered indexes [#9529](https://github.com/pingcap/tiflash/issues/9529) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - Add a check to verify whether the target cluster is an empty cluster for full restore [#35744](https://github.com/pingcap/tidb/issues/35744) @[3pointer](https://github.com/3pointer) + - Add a check to verify whether the target cluster contains a table with the same name for non-full restore [#55087](https://github.com/pingcap/tidb/issues/55087) @[RidRisR](https://github.com/RidRisR) + - Except for the `br log restore` subcommand, all other `br log` subcommands support skipping the loading of the TiDB `domain` data structure to reduce memory consumption [#52088](https://github.com/pingcap/tidb/issues/52088) @[Leavrth](https://github.com/Leavrth) + - Disable the table-level checksum calculation during full backups by default (`--checksum=false`) to improve backup performance [#56373](https://github.com/pingcap/tidb/issues/56373) @[Tristan1900](https://github.com/Tristan1900) + + + TiDB Lightning + + - Add a row width check when parsing CSV files to prevent OOM issues [#58590](https://github.com/pingcap/tidb/issues/58590) @[D3Hunter](https://github.com/D3Hunter) + +## Bug fixes + ++ TiDB + + - Fix the issue that using subqueries after the `NATURAL JOIN` or `USING` clause might result in errors [#53766](https://github.com/pingcap/tidb/issues/53766) @[dash12653](https://github.com/dash12653) + - Fix the issue that the `CAST` function does not support explicitly setting the character set [#55677](https://github.com/pingcap/tidb/issues/55677) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that the `LOAD DATA ... REPLACE INTO` operation causes data inconsistency [#56408](https://github.com/pingcap/tidb/issues/56408) @[fzzf678](https://github.com/fzzf678) + - Fix the issue that TiDB does not check the index length limitation when executing `ADD INDEX` [#56930](https://github.com/pingcap/tidb/issues/56930) @[fzzf678](https://github.com/fzzf678) + - Fix the issue of illegal memory access that might occur when a Common Table Expression (CTE) has multiple data consumers and one consumer exits without reading any data [#55881](https://github.com/pingcap/tidb/issues/55881) @[windtalker](https://github.com/windtalker) + - Fix the issue that some predicates might be lost when constructing `IndexMerge` [#58476](https://github.com/pingcap/tidb/issues/58476) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that converting data from the `BIT` type to the `CHAR` type might cause TiKV panics [#56494](https://github.com/pingcap/tidb/issues/56494) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that using variables or parameters in the `CREATE VIEW` statement does not report errors [#53176](https://github.com/pingcap/tidb/issues/53176) @[mjonss](https://github.com/mjonss) + - Fix the issue that unreleased session resources might lead to memory leaks [#56271](https://github.com/pingcap/tidb/issues/56271) @[lance6716](https://github.com/lance6716) + - Fix the issue that executing `ADD INDEX` might fail after modifying the PD member in the distributed execution framework [#48680](https://github.com/pingcap/tidb/issues/48680) @[lance6716](https://github.com/lance6716) + - Fix the issue that using `ORDER BY` when querying `cluster_slow_query table` might generate unordered results [#51723](https://github.com/pingcap/tidb/issues/51723) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that stale read does not strictly verify the timestamp of the read operation, resulting in a small probability of affecting the consistency of the transaction when an offset exists between the TSO and the real physical time [#56809](https://github.com/pingcap/tidb/issues/56809) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that the performance of querying `INFORMATION_SCHEMA.columns` degrades [#58184](https://github.com/pingcap/tidb/issues/58184) @[lance6716](https://github.com/lance6716) + - Fix the issue that the `INSERT ... ON DUPLICATE KEY` statement is not compatible with `mysql_insert_id` [#55965](https://github.com/pingcap/tidb/issues/55965) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the optimizer incorrectly estimates the number of rows as 1 when accessing a unique index with the query condition `column IS NULL` [#56116](https://github.com/pingcap/tidb/issues/56116) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that part of the memory of the `IndexLookUp` operator is not tracked [#56440](https://github.com/pingcap/tidb/issues/56440) @[wshwsh12](https://github.com/wshwsh12) + - Fix the potential data race issue that might occur in TiDB's internal coroutine [#57798](https://github.com/pingcap/tidb/issues/57798) [#56053](https://github.com/pingcap/tidb/issues/56053) @[fishiu](https://github.com/fishiu) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the `read_from_storage` hint might not take effect when the query has an available Index Merge execution plan [#56217](https://github.com/pingcap/tidb/issues/56217) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that execution plan bindings cannot be created for the multi-table `DELETE` statement with aliases [#56726](https://github.com/pingcap/tidb/issues/56726) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that `INDEX_HASH_JOIN` might hang during an abnormal exit [#54055](https://github.com/pingcap/tidb/issues/54055) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that two DDL Owners might exist at the same time [#54689](https://github.com/pingcap/tidb/issues/54689) @[joccau](https://github.com/joccau) + - Fix the issue that when querying the `information_schema.cluster_slow_query` table, if the time filter is not added, only the latest slow log file is queried [#56100](https://github.com/pingcap/tidb/issues/56100) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that `duplicate entry` might occur when adding unique indexes [#56161](https://github.com/pingcap/tidb/issues/56161) @[tangenta](https://github.com/tangenta) + - Fix the issue that the error message is incorrect in certain type conversion errors [#41730](https://github.com/pingcap/tidb/issues/41730) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the CTE defined in `VIEW` is incorrectly inlined [#56582](https://github.com/pingcap/tidb/issues/56582) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that the `UPDATE` statement incorrectly updates values of the `ENUM` type [#56832](https://github.com/pingcap/tidb/issues/56832) @[xhebox](https://github.com/xhebox) + - Fix the issue that executing the `UPDATE` statement after adding a `DATE` column results in the error `Incorrect date value: '0000-00-00'` in some cases [#59047](https://github.com/pingcap/tidb/issues/59047) @[mjonss](https://github.com/mjonss) + - Fix the issue that in the Prepare protocol, an error occurs when the client uses a non-UTF8 character set [#58870](https://github.com/pingcap/tidb/issues/58870) @[xhebox](https://github.com/xhebox) + - Fix the issue that querying temporary tables might trigger unexpected TiKV requests in some cases [#58875](https://github.com/pingcap/tidb/issues/58875) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the `ONLY_FULL_GROUP_BY` setting does not take effect on statements in views [#53175](https://github.com/pingcap/tidb/issues/53175) @[mjonss](https://github.com/mjonss) + - Fix the issue that querying partitioned tables using an `IN` condition containing a mismatched value type and a type conversion error leads to incorrect query results [#54746](https://github.com/pingcap/tidb/issues/54746) @[mjonss](https://github.com/mjonss) + - Fix the issue that querying slow logs might fail when certain fields contain empty values [#58147](https://github.com/pingcap/tidb/issues/58147) @[yibin87](https://github.com/yibin87) + - Fix the issue that the `RADIANS()` function computes values in an incorrect order [#57671](https://github.com/pingcap/tidb/issues/57671) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that the default value of the `BIT` column is incorrect [#57301](https://github.com/pingcap/tidb/issues/57301) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that an inline error might occur if a CTE contains `ORDER BY`, `LIMIT`, or `SELECT DISTINCT` clauses and is referenced by the recursive part of another CTE [#56603](https://github.com/pingcap/tidb/issues/56603) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that the timeout that occurs when loading statistics synchronically might not be handled correctly [#57710](https://github.com/pingcap/tidb/issues/57710) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that an incorrect database name might be returned when parsing the database name in a CTE [#54582](https://github.com/pingcap/tidb/issues/54582) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that TiDB might panic during startup due to invalid data binding [#58016](https://github.com/pingcap/tidb/issues/58016) @[qw4990](https://github.com/qw4990) + - Fix the issue that cost estimation might generate invalid INF/NaN values in certain extreme cases, which could lead to incorrect Join Reorder results [#56704](https://github.com/pingcap/tidb/issues/56704) @[winoros](https://github.com/winoros) + - Fix the issue that loading statistics manually might fail when the statistics file contains null values [#53966](https://github.com/pingcap/tidb/issues/53966) @[King-Dylan](https://github.com/King-Dylan) + - Fix the issue that creating two views with the same name does not report an error [#58769](https://github.com/pingcap/tidb/issues/58769) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that when a virtual generated column's dependencies contain a column with the `ON UPDATE` attribute, the data of the updated row and its index data might be inconsistent [#56829](https://github.com/pingcap/tidb/issues/56829) @[joechenrh](https://github.com/joechenrh) + - Fix the issue that the `INFORMATION_SCHEMA.TABLES` system table returns incorrect results [#57345](https://github.com/pingcap/tidb/issues/57345) @[tangenta](https://github.com/tangenta) + ++ TiKV + + - Fix the issue that Follower Read might read stale data [#17018](https://github.com/tikv/tikv/issues/17018) @[glorv](https://github.com/glorv) + - Fix the issue that TiKV might panic when destroying a peer [#18005](https://github.com/tikv/tikv/issues/18005) @[glorv](https://github.com/glorv) + - Fix the issue that time rollback might cause abnormal RocksDB flow control, leading to performance jitter [#17995](https://github.com/tikv/tikv/issues/17995) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that disk stalls might prevent leader migration, leading to performance jitter [#17363](https://github.com/tikv/tikv/issues/17363) @[hhwyt](https://github.com/hhwyt) + - Fix the issue that the latest written data might not be readable when only one-phase commit (1PC) is enabled and Async Commit is not enabled [#18117](https://github.com/tikv/tikv/issues/18117) @[zyguan](https://github.com/zyguan) + - Fix the issue that a deadlock might occur when GC Worker is under heavy load [#18214](https://github.com/tikv/tikv/issues/18214) @[zyguan](https://github.com/zyguan) + - Fix the issue that the **Storage async write duration** monitoring metric on the TiKV panel in Grafana is inaccurate [#17579](https://github.com/tikv/tikv/issues/17579) @[overvenus](https://github.com/overvenus) + - Fix the issue that TiKV might panic when executing queries containing `RADIANS()` or `DEGREES()` functions [#17852](https://github.com/tikv/tikv/issues/17852) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that merging Regions might cause TiKV to panic in rare cases [#17840](https://github.com/tikv/tikv/issues/17840) @[glorv](https://github.com/glorv) + - Fix the issue that the leader could not be quickly elected after Region split [#17602](https://github.com/tikv/tikv/issues/17602) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that encoding might fail when processing GBK/GB18030 encoded data [#17618](https://github.com/tikv/tikv/issues/17618) @[CbcWestwolf](https://github.com/CbcWestwolf) + ++ PD + + - Fix the issue that memory leaks might occur when allocating TSOs [#9004](https://github.com/tikv/pd/issues/9004) @[rleungx](https://github.com/rleungx) + - Fix the issue that the `tidb_enable_tso_follower_proxy` system variable might not take effect [#8947](https://github.com/tikv/pd/issues/8947) @[JmPotato](https://github.com/JmPotato) + - Fix a potential issue that might cause PD to panic [#8915](https://github.com/tikv/pd/issues/8915) @[bufferflies](https://github.com/bufferflies) + - Fix the issue that memory leaks might occur in long-running clusters [#9047](https://github.com/tikv/pd/issues/9047) @[bufferflies](https://github.com/bufferflies) + - Fix the issue that a PD node might still generate TSOs even when it is not the Leader [#9051](https://github.com/tikv/pd/issues/9051) @[rleungx](https://github.com/rleungx) + - Fix the issue that Region syncer might not exit in time during the PD Leader switch [#9017](https://github.com/tikv/pd/issues/9017) @[rleungx](https://github.com/rleungx) + - Fix the issue that when creating `evict-leader-scheduler` or `grant-leader-scheduler` encounters an error, the error message is not returned to pd-ctl [#8759](https://github.com/tikv/pd/issues/8759) @[okJiang](https://github.com/okJiang) + - Fix the memory leak issue in hotspot cache [#8698](https://github.com/tikv/pd/issues/8698) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that PD's Region API cannot be requested when a large number of Regions exist [#55872](https://github.com/pingcap/tidb/issues/55872) @[rleungx](https://github.com/rleungx) + - Fix the issue that `evict-leader-scheduler` fails to work properly when it is repeatedly created with the same Store ID [#8756](https://github.com/tikv/pd/issues/8756) @[okJiang](https://github.com/okJiang) + - Upgrade the version of Gin Web Framework from v1.9.1 to v1.10.0 to fix potential security vulnerabilities [#8643](https://github.com/tikv/pd/issues/8643) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that when using a wrong parameter in `evict-leader-scheduler`, PD does not report errors correctly and some schedulers are unavailable [#8619](https://github.com/tikv/pd/issues/8619) @[rleungx](https://github.com/rleungx) + - Fix the memory leak issue in label statistics [#8700](https://github.com/tikv/pd/issues/8700) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that TiDB Dashboard cannot read PD `trace` data correctly [#7253](https://github.com/tikv/pd/issues/7253) @[nolouch](https://github.com/nolouch) + - Fix the memory leak issue in Region statistics [#8710](https://github.com/tikv/pd/issues/8710) @[rleungx](https://github.com/rleungx) + - Fix the issue that PD cannot quickly re-elect a leader during etcd leader transition [#8823](https://github.com/tikv/pd/issues/8823) @[rleungx](https://github.com/rleungx) + ++ TiFlash + + - Fix the issue that the `SUBSTRING()` function does not support the `pos` and `len` arguments for certain integer types, causing query errors [#9473](https://github.com/pingcap/tiflash/issues/9473) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that some JSON functions unsupported by TiFlash are pushed down to TiFlash [#9444](https://github.com/pingcap/tiflash/issues/9444) @[windtalker](https://github.com/windtalker) + - Fix the issue that the `SUBSTRING()` function returns incorrect results when the second parameter is negative [#9604](https://github.com/pingcap/tiflash/issues/9604) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that `LPAD()` and `RPAD()` functions return incorrect results in some cases [#9465](https://github.com/pingcap/tiflash/issues/9465) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that executing `DROP TABLE` on large tables might cause TiFlash OOM [#9437](https://github.com/pingcap/tiflash/issues/9437) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash fails to start due to a division by zero error when retrieving the number of CPU cores [#9212](https://github.com/pingcap/tiflash/issues/9212) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that TiFlash might maintain high memory usage after importing large amounts of data [#9812](https://github.com/pingcap/tiflash/issues/9812) @[CalvinNeo](https://github.com/CalvinNeo) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that BR fails to restore due to getting the `rpcClient is idle` error when sending requests to TiKV [#58845](https://github.com/pingcap/tidb/issues/58845) @[Tristan1900](https://github.com/Tristan1900) + - Fix the issue that the `status` field is missing in the result when querying log backup tasks using `br log status --json` [#57959](https://github.com/pingcap/tidb/issues/57959) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that PD Leader I/O latency during log backup might increase checkpoint latency [#58574](https://github.com/pingcap/tidb/issues/58574) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that the `tiup br restore` command omits checking whether the target cluster table already exists during database or table restoration, which might overwrite existing tables [#58168](https://github.com/pingcap/tidb/issues/58168) @[RidRisR](https://github.com/RidRisR) + - Fix the issue that log backup might unexpectedly enter a paused state when the advancer owner switches [#58031](https://github.com/pingcap/tidb/issues/58031) @[3pointer](https://github.com/3pointer) + - Fix the issue that log backups cannot resolve residual locks promptly, causing the checkpoint to fail to advance [#57134](https://github.com/pingcap/tidb/issues/57134) @[3pointer](https://github.com/3pointer) + - Fix the issue that BR integration test cases are unstable, and add a new test case to simulate snapshot or log backup file corruption [#53835](https://github.com/pingcap/tidb/issues/53835) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that logs might print out encrypted information [#57585](https://github.com/pingcap/tidb/issues/57585) @[kennytm](https://github.com/kennytm) + - Fix the issue that PITR tasks might return the `Information schema is out of date` error when there are a large number of tables in the cluster but the actual data size is small [#57743](https://github.com/pingcap/tidb/issues/57743) @[Tristan1900](https://github.com/Tristan1900) + + + TiCDC + + - Fix the issue that TiCDC uses incorrect table names for filtering during `RENAME TABLE` operations [#11946](https://github.com/pingcap/tiflow/issues/11946) @[wk989898](https://github.com/wk989898) + - Fix the issue that TiCDC reports errors when replicating `default NULL` SQL statements via the Avro protocol [#11994](https://github.com/pingcap/tiflow/issues/11994) @[wk989898](https://github.com/wk989898) + - Fix the issue that TiCDC fails to properly connect to PD after PD scale-in [#12004](https://github.com/pingcap/tiflow/issues/12004) @[lidezhu](https://github.com/lidezhu) + - Fix the issue that Initial Scan is not canceled after the changefeed is stopped or deleted [#11638](https://github.com/pingcap/tiflow/issues/11638) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that after the default value of a newly added column in the upstream is changed from `NOT NULL` to `NULL`, the default values of that column in the downstream are incorrect [#12037](https://github.com/pingcap/tiflow/issues/12037) @[wk989898](https://github.com/wk989898) + - Fix the issue that using the `--overwrite-checkpoint-ts` parameter in the `changefeed pause` command might cause the changefeed to be stuck [#12055](https://github.com/pingcap/tiflow/issues/12055) @[hongyunyan](https://github.com/hongyunyan) + - Fix the issue that TiCDC might panic when replicating `CREATE TABLE IF NOT EXISTS` or `CREATE DATABASE IF NOT EXISTS` statements [#11839](https://github.com/pingcap/tiflow/issues/11839) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that TiCDC might report an error when replicating a `TRUNCATE TABLE` DDL on a table without valid index [#11765](https://github.com/pingcap/tiflow/issues/11765) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that TiCDC mistakenly discards DDL tasks when the schema versions of DDL tasks become non-incremental during TiDB DDL owner changes [#11714](https://github.com/pingcap/tiflow/issues/11714) @[wlwilliamx](https://github.com/wlwilliamx) + - Fix the issue that the changefeed might get stuck after new TiKV nodes are added to the cluster [#11766](https://github.com/pingcap/tiflow/issues/11766) @[lidezhu](https://github.com/lidezhu) + - Fix the issue that out-of-order messages resent by the Sarama client cause Kafka message order to be incorrect [#11935](https://github.com/pingcap/tiflow/issues/11935) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that the Resolved TS latency monitoring in the Puller module displays incorrect values [#11561](https://github.com/pingcap/tiflow/issues/11561) @[wlwilliamx](https://github.com/wlwilliamx) + - Fix the issue that the redo module fails to properly report errors [#11744](https://github.com/pingcap/tiflow/issues/11744) @[CharlesCheung96](https://github.com/CharlesCheung96) + + + TiDB Data Migration (DM) + + - Fix the issue that multiple DM-master nodes might simultaneously become leaders, leading to data inconsistency [#11602](https://github.com/pingcap/tiflow/issues/11602) @[GMHDBJD](https://github.com/GMHDBJD) + - Fix the issue that connecting to MySQL 8.0 fails when the password length exceeds 19 characters [#11603](https://github.com/pingcap/tiflow/issues/11603) @[fishiu](https://github.com/fishiu) + - Fix the issue that the pre-check of `start-task` fails when both TLS and `shard-mode` are configured [#11842](https://github.com/pingcap/tiflow/issues/11842) @[sunxiaoguang](https://github.com/sunxiaoguang) + + + TiDB Lightning + + - Fix the issue that logs are not properly desensitized [#59086](https://github.com/pingcap/tidb/issues/59086) @[GMHDBJD](https://github.com/GMHDBJD) + - Fix the issue that the lack of caching in the encoding phase causes performance regression [#56705](https://github.com/pingcap/tidb/issues/56705) @[OliverS929](https://github.com/OliverS929) + - Fix the issue that the performance degrades when importing data from a cloud storage in high-concurrency scenarios [#57413](https://github.com/pingcap/tidb/issues/57413) @[xuanyu66](https://github.com/xuanyu66) + - Fix the issue that TiDB Lightning does not automatically retry when encountering `Lock wait timeout` errors during metadata updates [#53042](https://github.com/pingcap/tidb/issues/53042) @[guoshouyan](https://github.com/guoshouyan) + - Fix the issue that TiDB Lightning fails to receive oversized messages sent from TiKV [#56114](https://github.com/pingcap/tidb/issues/56114) @[fishiu](https://github.com/fishiu) + - Fix the issue that the error report output is truncated when importing data using TiDB Lightning [#58085](https://github.com/pingcap/tidb/issues/58085) @[lance6716](https://github.com/lance6716) + + + Dumpling + + - Fix the issue that Dumpling fails to retry properly when receiving a 503 error from Google Cloud Storage (GCS) [#56127](https://github.com/pingcap/tidb/issues/56127) @[OliverS929](https://github.com/OliverS929) diff --git a/releases/release-6.5.2.md b/releases/release-6.5.2.md index 53a6eded2e74a..3825d7a6be554 100644 --- a/releases/release-6.5.2.md +++ b/releases/release-6.5.2.md @@ -9,7 +9,7 @@ Release date: April 21, 2023 TiDB version: 6.5.2 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.5.2#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) ## Compatibility changes @@ -65,7 +65,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Fix the issue that full index scans might cause errors when prepared plan cache is enabled [#42150](https://github.com/pingcap/tidb/issues/42150) @[fzzf678](https://github.com/fzzf678) - Fix the issue that IndexMerge might produce incorrect results when prepare plan cache is enabled [#41828](https://github.com/pingcap/tidb/issues/41828) @[qw4990](https://github.com/qw4990) - Fix the issue that the configuration of `max_prepared_stmt_count` does not take effect [#39735](https://github.com/pingcap/tidb/issues/39735) @[xuyifangreeneyes](https://github.com/xuyifangreeneyes) - - Fix the issue that IndexMerge might produce incorrect results when prepare plan cache is enabled [#41828](https://github.com/pingcap/tidb/issues/41828) @[qw4990](https://github.com/qw4990) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that global memory control might incorrectly kill SQL statements with memory usage less than `tidb_server_memory_limit_sess_min_size` [#42662](https://github.com/pingcap/tidb/issues/42662) @[XuHuaiyu](https://github.com/XuHuaiyu) - Fix the issue that Index Join might cause panic in dynamic trimming mode of partition tables [#40596](https://github.com/pingcap/tidb/issues/40596) @[tiancaiamao](https://github.com/tiancaiamao) + TiKV @@ -94,7 +94,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- + TiCDC - - Fix the issue that the partition separator does not work when TiCDC replicates data to object storage [#8581](https://github.com/pingcap/tiflow/issues/8581) @[CharlesCheung96](https://github.com/CharlesCheung96) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that the partition separator does not work when TiCDC replicates data to object storage [#8581](https://github.com/pingcap/tiflow/issues/8581) @[CharlesCheung96](https://github.com/CharlesCheung96) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that table scheduling might cause data loss when TiCDC replicates data to object storage [#8256](https://github.com/pingcap/tiflow/issues/8256) @[zhaoxinyu](https://github.com/zhaoxinyu) - Fix the issue that the replication gets stuck due to non-reentrant DDL statements [#8662](https://github.com/pingcap/tiflow/issues/8662) @[hicqu](https://github.com/hicqu) - Fix the issue that TiCDC scaling might cause data loss when TiCDC replicates data to object storage [#8666](https://github.com/pingcap/tiflow/issues/8666) @[CharlesCheung96](https://github.com/CharlesCheung96) diff --git a/releases/release-6.5.3.md b/releases/release-6.5.3.md index 8a2afd9bb1488..f1ca8f2605be4 100644 --- a/releases/release-6.5.3.md +++ b/releases/release-6.5.3.md @@ -9,7 +9,13 @@ Release date: June 14, 2023 TiDB version: 6.5.3 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.5.3#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) + +## Compatibility changes + +### Behavior changes + +- When processing update event, TiCDC splits an event into delete and insert events if the primary key or non-null unique index value is modified in the event. For more information, see [documentation](/ticdc/ticdc-split-update-behavior.md#transactions-containing-a-single-update-change). ## Improvements @@ -35,11 +41,11 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Optimize the way TiCDC handles DDLs so that DDLs do not block the use of other unrelated DML Events, and reduce memory usage [#8106](https://github.com/pingcap/tiflow/issues/8106) @[asddongmen](https://github.com/asddongmen) - Optimize the Decoder interface and add a new method `AddKeyValue` [#8861](https://github.com/pingcap/tiflow/issues/8861) @[3AceShowHand](https://github.com/3AceShowHand) - Optimize the directory structure when DDL events occur in the scenario of replicating data to object storage [#8890](https://github.com/pingcap/tiflow/issues/8890) @[CharlesCheung96](https://github.com/CharlesCheung96) - - Support replicating data to the Kafka-on-Pulsar downstream [#8892](https://github.com/pingcap/tiflow/issues/8892) @[hi-rustin](https://github.com/hi-rustin) - - Support using the OAuth protocol for validation when replicating data to Kafka [#8865](https://github.com/pingcap/tiflow/issues/8865) @[hi-rustin](https://github.com/hi-rustin) + - Support replicating data to the Kafka-on-Pulsar downstream [#8892](https://github.com/pingcap/tiflow/issues/8892) @[hi-rustin](https://github.com/Rustin170506) + - Support using the OAuth protocol for validation when replicating data to Kafka [#8865](https://github.com/pingcap/tiflow/issues/8865) @[hi-rustin](https://github.com/Rustin170506) - Optimize the way TiCDC handles the `UPDATE` statement during data replication using the Avro or CSV protocol, by splitting `UPDATE` into `DELETE` and `INSERT` statements, so that you can get the old value from the `DELETE` statement [#9086](https://github.com/pingcap/tiflow/issues/9086) @[3AceShowHand](https://github.com/3AceShowHand) - - Add a configuration item `insecure-skip-verify` to control whether to set the authentication algorithm in the scenario of enabling TLS [#8867](https://github.com/pingcap/tiflow/issues/8867) @[hi-rustin](https://github.com/hi-rustin) - - Optimize DDL replication operations to mitigate the impact of DDL operations on downstream latency [#8686](https://github.com/pingcap/tiflow/issues/8686) @[hi-rustin](https://github.com/hi-rustin) + - Add a configuration item `insecure-skip-verify` to control whether to set the authentication algorithm in the scenario of enabling TLS [#8867](https://github.com/pingcap/tiflow/issues/8867) @[hi-rustin](https://github.com/Rustin170506) + - Optimize DDL replication operations to mitigate the impact of DDL operations on downstream latency [#8686](https://github.com/pingcap/tiflow/issues/8686) @[hi-rustin](https://github.com/Rustin170506) - Optimize the method of setting GC TLS for the upstream when the TiCDC replication task fails [#8403](https://github.com/pingcap/tiflow/issues/8403) @[charleszheng44](https://github.com/charleszheng44) + TiDB Binlog @@ -111,12 +117,12 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Fix an OOM issue that might occur when there are as many as 50,000 tables [#7872](https://github.com/pingcap/tiflow/issues/7872) @[sdojjy](https://github.com/sdojjy) - Fix the issue that TiCDC gets stuck when an OOM occurs in upstream TiDB [#8561](https://github.com/pingcap/tiflow/issues/8561) @[overvenus](https://github.com/overvenus) - Fix the issue that TiCDC gets stuck when PD fails such as network isolation or PD Owner node reboot [#8808](https://github.com/pingcap/tiflow/issues/8808) [#8812](https://github.com/pingcap/tiflow/issues/8812) [#8877](https://github.com/pingcap/tiflow/issues/8877) @[asddongmen](https://github.com/asddongmen) - - Fix the issue of TiCDC time zone setting [#8798](https://github.com/pingcap/tiflow/issues/8798) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue of TiCDC time zone setting [#8798](https://github.com/pingcap/tiflow/issues/8798) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that checkpoint lag increases when one of the upstream TiKV nodes crashes [#8858](https://github.com/pingcap/tiflow/issues/8858) @[hicqu](https://github.com/hicqu) - Fix the issue that when replicating data to downstream MySQL, a replication error occurs after the `FLASHBACK CLUSTER TO TIMESTAMP` statement is executed in the upstream TiDB [#8040](https://github.com/pingcap/tiflow/issues/8040) @[asddongmen](https://github.com/asddongmen) - Fix the issue that when replicating data to object storage, the `EXCHANGE PARTITION` operation in the upstream cannot be properly replicated to the downstream [#8914](https://github.com/pingcap/tiflow/issues/8914) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the OOM issue caused by excessive memory usage of the sorter component in some special scenarios [#8974](https://github.com/pingcap/tiflow/issues/8974) @[hicqu](https://github.com/hicqu) - - Fix the issue that when the downstream is Kafka, TiCDC queries the downstream metadata too frequently and causes excessive workload in the downstream [#8957](https://github.com/pingcap/tiflow/issues/8957) [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that when the downstream is Kafka, TiCDC queries the downstream metadata too frequently and causes excessive workload in the downstream [#8957](https://github.com/pingcap/tiflow/issues/8957) [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that when a replication error occurs due to an oversized Kafka message, the message body is recorded in the log [#9031](https://github.com/pingcap/tiflow/issues/9031) @[darraes](https://github.com/darraes) - Fix the TiCDC node panic that occurs when the downstream Kafka sinks are rolling restarted [#9023](https://github.com/pingcap/tiflow/issues/9023) @[asddongmen](https://github.com/asddongmen) - Fix the issue that when replicating data to storage services, the JSON file corresponding to downstream DDL statements does not record the default values of table fields [#9066](https://github.com/pingcap/tiflow/issues/9066) @[CharlesCheung96](https://github.com/CharlesCheung96) diff --git a/releases/release-6.5.4.md b/releases/release-6.5.4.md index 09e21ddf5449a..772a97dd52117 100644 --- a/releases/release-6.5.4.md +++ b/releases/release-6.5.4.md @@ -9,13 +9,17 @@ Release date: August 28, 2023 TiDB version: 6.5.4 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.5.4#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) ## Compatibility changes - To fix the issue that TiDB consumes too much memory when using `Cursor Fetch` to fetch a large result set, TiDB automatically writes the result set to the disk to release memory [#43233](https://github.com/pingcap/tidb/issues/43233) @[YangKeao](https://github.com/YangKeao) - Disable periodic compaction of RocksDB by default, so that the default behavior of TiKV RocksDB is now consistent with that in versions before v6.5.0. This change prevents potential performance impact caused by a significant number of compactions after upgrading. In addition, TiKV introduces two new configuration items [`rocksdb.[defaultcf|writecf|lockcf].periodic-compaction-seconds`](https://docs.pingcap.com/tidb/v6.5/tikv-configuration-file#periodic-compaction-seconds-new-in-v654) and [`rocksdb.[defaultcf|writecf|lockcf].ttl`](https://docs.pingcap.com/tidb/v6.5/tikv-configuration-file#ttl-new-in-v654), enabling you to manually configure periodic compaction of RocksDB [#15355](https://github.com/tikv/tikv/issues/15355) @[LykxSassinator](https://github.com/LykxSassinator) +### Behavior changes + +- For transactions containing multiple changes, if the primary key or non-null unique index value is modified in the update event, TiCDC splits an event into delete and insert events and ensures that all events follow the sequence of delete events preceding insert events. For more information, see [documentation](/ticdc/ticdc-split-update-behavior.md#transactions-containing-multiple-update-changes). + ## Improvements + TiDB @@ -30,6 +34,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Add the `Max gap of safe-ts` and `Min safe ts region` metrics and introduce the `tikv-ctl get-region-read-progress` command to better observe and diagnose the status of resolved-ts and safe-ts [#15082](https://github.com/tikv/tikv/issues/15082) @[ekexium](https://github.com/ekexium) - Expose some RocksDB configurations in TiKV that allow users to disable features such as TTL and periodic compaction [#14873](https://github.com/tikv/tikv/issues/14873) @[LykxSassinator](https://github.com/LykxSassinator) - Avoid holding mutex when writing Titan manifest files to prevent affecting other threads [#15351](https://github.com/tikv/tikv/issues/15351) @[Connor1996](https://github.com/Connor1996) + - Optimize the compaction mechanism: when a Region is split, if there is no key to split, a compaction is triggered to eliminate excessive MVCC versions [#15282](https://github.com/tikv/tikv/issues/15282) @[SpadeA-Tang](https://github.com/SpadeA-Tang) + PD @@ -79,7 +84,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Fix the issue that using CTEs and correlated subqueries simultaneously might result in incorrect query results or panic [#44649](https://github.com/pingcap/tidb/issues/44649) [#38170](https://github.com/pingcap/tidb/issues/38170) [#44774](https://github.com/pingcap/tidb/issues/44774) @[winoros](https://github.com/winoros) @[guo-shaoge](https://github.com/guo-shaoge) - Fix the issue that TTL tasks cannot trigger statistics updates in time [#40109](https://github.com/pingcap/tidb/issues/40109) @[YangKeao](https://github.com/YangKeao) - Fix the issue that the GC Resolve Locks step might miss some pessimistic locks [#45134](https://github.com/pingcap/tidb/issues/45134) @[MyonKeminta](https://github.com/MyonKeminta) - - Fix the potential memory leak issue in memory tracker [#44612](https://github.com/pingcap/tidb/issues/44612) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that memory leaks and duration continues to increase when connecting to TiDB using the binary protocol and executing a large number of `PREPARE` and `EXECUTE` statements [#44612](https://github.com/pingcap/tidb/issues/44612) @[wshwsh12](https://github.com/wshwsh12) - Fix the issue that the data length in the `QUERY` column of the `INFORMATION_SCHEMA.DDL_JOBS` table might exceed the column definition [#42440](https://github.com/pingcap/tidb/issues/42440) @[tiancaiamao](https://github.com/tiancaiamao) - Fix the PD OOM issue when there is a large number of Regions but the table ID cannot be pushed down when querying some virtual tables using `Prepare` or `Execute` [#39605](https://github.com/pingcap/tidb/issues/39605) @[djshow832](https://github.com/djshow832) - Fix the issue that statistics auto-collection might not trigger correctly on a partitioned table after adding a new index to the partitioned table [#41638](https://github.com/pingcap/tidb/issues/41638) @[xuyifangreeneyes](https://github.com/xuyifangreeneyes) @@ -107,7 +112,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Fix the issue that killing a connection might cause go coroutine leaks [#46034](https://github.com/pingcap/tidb/issues/46034) @[pingyu](https://github.com/pingyu) - Fix the issue that the `tmp-storage-quota` configuration does not take effect [#45161](https://github.com/pingcap/tidb/issues/45161) [#26806](https://github.com/pingcap/tidb/issues/26806) @[wshwsh12](https://github.com/wshwsh12) - Fix the issue that TiFlash replicas might be unavailable when a TiFlash node is down in the cluster [#38484](https://github.com/pingcap/tidb/issues/38484) @[hehechen](https://github.com/hehechen) - - Fix the issue that TiDB crashes due to possible data race when reading and writing `Config.Lables` concurrently [#45561](https://github.com/pingcap/tidb/issues/45561) @[genliqi](https://github.com/gengliqi) + - Fix the issue that TiDB crashes due to possible data race when reading and writing `Config.Labels` concurrently [#45561](https://github.com/pingcap/tidb/issues/45561) @[genliqi](https://github.com/gengliqi) - Fix the issue that the client-go regularly updating `min-resolved-ts` might cause PD OOM when the cluster is large [#46664](https://github.com/pingcap/tidb/issues/46664) @[HuSharp](https://github.com/HuSharp) + TiKV @@ -127,7 +132,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Fix the issue that when etcd is already started but the client has not yet connected to it, calling the client might cause PD to panic [#6860](https://github.com/tikv/pd/issues/6860) @[HuSharp](https://github.com/HuSharp) - Fix the issue that a leader cannot exit for a long time [#6918](https://github.com/tikv/pd/issues/6918) @[bufferflies](https://github.com/bufferflies) - - Fix the issue that when the placement rule uses `LOCATION_LABLES`, SQL and the Rule Checker are not compatible [#38605](https://github.com/pingcap/tidb/issues/38605) @[nolouch](https://github.com/nolouch) + - Fix the issue that when the placement rule uses `LOCATION_LABELS`, SQL and the Rule Checker are not compatible [#38605](https://github.com/pingcap/tidb/issues/38605) @[nolouch](https://github.com/nolouch) - Fix the issue that PD might unexpectedly add multiple Learners to a Region [#5786](https://github.com/tikv/pd/issues/5786) @[HunDunDM](https://github.com/HunDunDM) - Fix the issue that unhealthy peers cannot be removed when rule checker selects peers [#6559](https://github.com/tikv/pd/issues/6559) @[nolouch](https://github.com/nolouch) - Fix the issue that failed learner peers in `unsafe recovery` are ignored in `auto-detect` mode [#6690](https://github.com/tikv/pd/issues/6690) @[v01dstar](https://github.com/v01dstar) @@ -161,7 +166,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Fix the issue that the replication task might get stuck when the downstream encounters a short-term failure [#9542](https://github.com/pingcap/tiflow/issues/9542) [#9272](https://github.com/pingcap/tiflow/issues/9272)[#9582](https://github.com/pingcap/tiflow/issues/9582) [#9592](https://github.com/pingcap/tiflow/issues/9592) @[hicqu](https://github.com/hicqu) - Fix the panic issue that might occur when the TiCDC node status changes [#9354](https://github.com/pingcap/tiflow/issues/9354) @[sdojjy](https://github.com/sdojjy) - Fix the issue that when Kafka Sink encounters errors it might indefinitely block changefeed progress [#9309](https://github.com/pingcap/tiflow/issues/9309) @[hicqu](https://github.com/hicqu) - - Fix the issue that when the downstream is Kafka, TiCDC queries the downstream metadata too frequently and causes excessive workload in the downstream [#8957](https://github.com/pingcap/tiflow/issues/8957) [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that when the downstream is Kafka, TiCDC queries the downstream metadata too frequently and causes excessive workload in the downstream [#8957](https://github.com/pingcap/tiflow/issues/8957) [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/Rustin170506) - Fix the data inconsistency issue that might occur when some TiCDC nodes are isolated from the network [#9344](https://github.com/pingcap/tiflow/issues/9344) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the issue that the replication task might get stuck when the redo log is enabled and there is an exception downstream [#9172](https://github.com/pingcap/tiflow/issues/9172) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the issue that changefeeds would fail due to the temporary unavailability of PD [#9294](https://github.com/pingcap/tiflow/issues/9294) @[asddongmen](https://github.com/asddongmen) diff --git a/releases/release-6.5.5.md b/releases/release-6.5.5.md index 83a37a37dec6b..3d653d2f92a75 100644 --- a/releases/release-6.5.5.md +++ b/releases/release-6.5.5.md @@ -9,7 +9,7 @@ Release date: September 21, 2023 TiDB version: 6.5.5 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.5.5#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) ## Improvements @@ -25,7 +25,6 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Improve stability of PITR checkpoint lag during leader transfers [#13638](https://github.com/tikv/tikv/issues/13638) @[YuJuncen](https://github.com/YuJuncen) - Add logs and monitoring metrics related to `safe-ts` [#15082](https://github.com/tikv/tikv/issues/15082) @[ekexium](https://github.com/ekexium) - Provide more logs and monitoring metrics for `resolved-ts` [#15082](https://github.com/tikv/tikv/issues/15082) @[ekexium](https://github.com/ekexium) - - Optimize the compaction mechanism: when a Region is split, if there is no key to split, a compaction is triggered to eliminate excessive MVCC versions [#15282](https://github.com/tikv/tikv/issues/15282) @[SpadeA-Tang](https://github.com/SpadeA-Tang) + Tools @@ -57,7 +56,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Fix the issue that restoring implicit primary keys by PITR might lead to conflicts [#46520](https://github.com/pingcap/tidb/issues/46520) @[3pointer](https://github.com/3pointer) - Fix the issue that an error occurs when PITR recovers the meta-kv [#46578](https://github.com/pingcap/tidb/issues/46578) @[Leavrth](https://github.com/Leavrth) - - Fix an error in BR integration test cases [#45561](https://github.com/pingcap/tidb/issues/46561) @[purelind](https://github.com/purelind) + - Fix an error in BR integration test cases [#46561](https://github.com/pingcap/tidb/issues/46561) @[purelind](https://github.com/purelind) - Fix the issue that PITR fails to restore data from GCS [#47022](https://github.com/pingcap/tidb/issues/47022) @[Leavrth](https://github.com/Leavrth) + TiCDC diff --git a/releases/release-6.5.6.md b/releases/release-6.5.6.md new file mode 100644 index 0000000000000..d7f075e2e43b9 --- /dev/null +++ b/releases/release-6.5.6.md @@ -0,0 +1,181 @@ +--- +title: TiDB 6.5.6 Release Notes +summary: Learn about the improvements and bug fixes in TiDB 6.5.6. +--- + +# TiDB 6.5.6 Release Notes + +Release date: December 7, 2023 + +TiDB version: 6.5.6 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) + +## Compatibility changes + +- Prohibit setting [`require_secure_transport`](https://docs.pingcap.com/tidb/v6.5/system-variables#require_secure_transport-new-in-v610) to `ON` in Security Enhanced Mode (SEM) to prevent potential connectivity issues for users [#47665](https://github.com/pingcap/tidb/issues/47665) @[tiancaiamao](https://github.com/tiancaiamao) +- Introduce the [`tidb_opt_enable_hash_join`](https://docs.pingcap.com/tidb/v6.5/system-variables#tidb_opt_enable_hash_join-new-in-v656) system variable to control whether the optimizer selects hash joins for tables [#46695](https://github.com/pingcap/tidb/issues/46695) @[coderplay](https://github.com/coderplay) +- After further testing, the default value of the TiCDC Changefeed configuration item [`case-sensitive`](/ticdc/ticdc-changefeed-config.md) is changed from `true` to `false`. This means that by default, table and database names in the TiCDC configuration file are case-insensitive [#10047](https://github.com/pingcap/tiflow/issues/10047) @[sdojjy](https://github.com/sdojjy) +- TiCDC Changefeed introduces the following new configuration items: + - [`sql-mode`](/ticdc/ticdc-changefeed-config.md): enables you to set the [SQL mode](/ticdc/ticdc-ddl.md#sql-mode) used by TiCDC to parse DDL statements when TiCDC replicates data [#9876](https://github.com/pingcap/tiflow/issues/9876) @[asddongmen](https://github.com/asddongmen) + - [`encoding-worker-num`](/ticdc/ticdc-changefeed-config.md) and [`flush-worker-num`](/ticdc/ticdc-changefeed-config.md): enables you to set different concurrency parameters for the redo module based on specifications of different machines [#10048](https://github.com/pingcap/tiflow/issues/10048) @[CharlesCheung96](https://github.com/CharlesCheung96) + - [`compression`](/ticdc/ticdc-changefeed-config.md): enables you to configure the compression behavior of redo log files [#10176](https://github.com/pingcap/tiflow/issues/10176) @[sdojjy](https://github.com/sdojjy) + - [`changefeed-error-stuck-duration`](/ticdc/ticdc-changefeed-config.md): enables you to set the duration for which the changefeed is allowed to automatically retry when internal errors or exceptions occur [#9875](https://github.com/pingcap/tiflow/issues/9875) @[asddongmen](https://github.com/asddongmen) + - [`sink.cloud-storage-config`](/ticdc/ticdc-changefeed-config.md): enables you to set the automatic cleanup of historical data when replicating data to object storage [#10109](https://github.com/pingcap/tiflow/issues/10109) @[CharlesCheung96](https://github.com/CharlesCheung96) + +## Improvements + ++ TiDB + + - Support the [`FLASHBACK CLUSTER TO TSO`](https://docs.pingcap.com/tidb/v6.5/sql-statement-flashback-cluster) syntax [#48372](https://github.com/pingcap/tidb/issues/48372) @[BornChanger](https://github.com/BornChanger) + ++ TiKV + + - Optimize memory usage of Resolver to prevent OOM [#15458](https://github.com/tikv/tikv/issues/15458) @[overvenus](https://github.com/overvenus) + - Eliminate LRUCache in Router objects to reduce memory usage and prevent OOM [#15430](https://github.com/tikv/tikv/issues/15430) @[Connor1996](https://github.com/Connor1996) + - Add the `alive` and `leak` monitoring dimensions for the `apply_router` and `raft_router` metrics [#15357](https://github.com/tikv/tikv/issues/15357) @[tonyxuqqi](https://github.com/tonyxuqqi) + ++ PD + + - Add monitoring metrics such as `Status` and `Sync Progress` for `DR Auto-Sync` on the Grafana dashboard [#6975](https://github.com/tikv/pd/issues/6975) @[disksing](https://github.com/disksing) + ++ Tools + + + Backup & Restore (BR) + + - During restoring a snapshot backup, BR retries when it encounters certain network errors [#48528](https://github.com/pingcap/tidb/issues/48528) @[Leavrth](https://github.com/Leavrth) + - Introduce a new integration test for Point-In-Time Recovery (PITR) in the `delete range` scenario, enhancing PITR stability [#47738](https://github.com/pingcap/tidb/issues/47738) @[Leavrth](https://github.com/Leavrth) + - Enable automatic retry of Region scatter during snapshot recovery when encountering timeout failures or cancellations of Region scatter [#47236](https://github.com/pingcap/tidb/issues/47236) @[Leavrth](https://github.com/Leavrth) + - BR can pause Region merging by setting the `merge-schedule-limit` configuration to `0` [#7148](https://github.com/tikv/pd/issues/7148) @[BornChanger](https://github.com/3pointer) + + + TiCDC + + - Support making TiCDC Canal-JSON content format [compatible with the content format of the official Canal output](https://docs.pingcap.com/tidb/v6.5/ticdc-canal-json#compatibility-with-the-official-canal) by setting `content-compatible=true` in the `sink-uri` configuration [#10106](https://github.com/pingcap/tiflow/issues/10106) @[3AceShowHand](https://github.com/3AceShowHand) + - Optimize the execution logic of replicating the `ADD INDEX` DDL operations to avoid blocking subsequent DML statements [#9644](https://github.com/pingcap/tiflow/issues/9644) @[sdojjy](https://github.com/sdojjy) + - Reduce the impact of TiCDC incremental scanning on upstream TiKV [#11390](https://github.com/tikv/tikv/issues/11390) @[hicqu](https://github.com/hicqu) + +## Bug fixes + ++ TiDB + + - Fix the issue that the chunk cannot be reused when the HashJoin operator performs probe [#48082](https://github.com/pingcap/tidb/issues/48082) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that `Duplicate entry` might occur when `AUTO_ID_CACHE=1` is set [#46444](https://github.com/pingcap/tidb/issues/46444) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the `TIDB_INLJ` hint does not take effect when joining two sub-queries [#46160](https://github.com/pingcap/tidb/issues/46160) @[qw4990](https://github.com/qw4990) + - Fix the issue that DDL operations might get stuck after TiDB is restarted [#46751](https://github.com/pingcap/tidb/issues/46751) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that DDL operations might get permanently blocked due to incorrect MDL handling [#46920](https://github.com/pingcap/tidb/issues/46920) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that the results of `MERGE_JOIN` are incorrect [#46580](https://github.com/pingcap/tidb/issues/46580) @[qw4990](https://github.com/qw4990) + - Fix the issue that the Sort operator might cause TiDB to crash during the spill process [#47538](https://github.com/pingcap/tidb/issues/47538) @[windtalker](https://github.com/windtalker) + - Fix the issue that the `cast(col)=range` condition causes FullScan when CAST has no precision loss [#45199](https://github.com/pingcap/tidb/issues/45199) @[AilinKid](https://github.com/AilinKid) + - Fix the panic issue of `batch-client` in `client-go` [#47691](https://github.com/pingcap/tidb/issues/47691) @[crazycs520](https://github.com/crazycs520) + - Prohibit split table operations on non-integer clustered indexes [#47350](https://github.com/pingcap/tidb/issues/47350) @[tangenta](https://github.com/tangenta) + - Fix the incompatibility issue between the behavior of prepared plan cache and non-prepared plan cache during time conversion [#42439](https://github.com/pingcap/tidb/issues/42439) @[qw4990](https://github.com/qw4990) + - Fix the issue that sometimes an index cannot be created for an empty table using ingest mode [#39641](https://github.com/pingcap/tidb/issues/39641) @[tangenta](https://github.com/tangenta) + - Fix the issue of not being able to detect data that does not comply with partition definitions during partition exchange [#46492](https://github.com/pingcap/tidb/issues/46492) @[mjonss](https://github.com/mjonss) + - Fix the issue that `GROUP_CONCAT` cannot parse the `ORDER BY` column [#41986](https://github.com/pingcap/tidb/issues/41986) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that HashCode is repeatedly calculated for deeply nested expressions, which causes high memory usage and OOM [#42788](https://github.com/pingcap/tidb/issues/42788) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that when Aggregation is pushed down through Union in MPP execution plans, the results are incorrect [#45850](https://github.com/pingcap/tidb/issues/45850) @[AilinKid](https://github.com/AilinKid) + - Fix the issue of incorrect memory usage estimation in `INDEX_LOOKUP_HASH_JOIN` [#47788](https://github.com/pingcap/tidb/issues/47788) @[SeaRise](https://github.com/SeaRise) + - Fix the issue that the zip file generated by `plan replayer` cannot be imported back into TiDB [#46474](https://github.com/pingcap/tidb/issues/46474) @[YangKeao](https://github.com/YangKeao) + - Fix the incorrect cost estimation caused by an excessively large `N` in `LIMIT N` [#43285](https://github.com/pingcap/tidb/issues/43285) @[qw4990](https://github.com/qw4990) + - Fix the panic issue that might occur when constructing TopN structure for statistics [#35948](https://github.com/pingcap/tidb/issues/35948) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that the result of `COUNT(INT)` calculated by MPP might be incorrect [#48643](https://github.com/pingcap/tidb/issues/48643) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that panic might occur when `tidb_enable_ordered_result_mode` is enabled [#45044](https://github.com/pingcap/tidb/issues/45044) @[qw4990](https://github.com/qw4990) + - Fix the issue that the optimizer mistakenly selects IndexFullScan to reduce sort introduced by window functions [#46177](https://github.com/pingcap/tidb/issues/46177) @[qw4990](https://github.com/qw4990) + - Fix the issue that the result might be incorrect when predicates are pushed down to common table expressions [#47881](https://github.com/pingcap/tidb/issues/47881) @[winoros](https://github.com/winoros) + - Fix the issue that executing `UNION ALL` with the DUAL table as the first subnode might cause an error [#48755](https://github.com/pingcap/tidb/issues/48755) @[winoros](https://github.com/winoros) + - Fix the issue that column pruning can cause panic in specific situations [#47331](https://github.com/pingcap/tidb/issues/47331) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue of possible syntax error when a common table expression (CTE) containing aggregate or window functions is referenced by other recursive CTEs [#47603](https://github.com/pingcap/tidb/issues/47603) [#47711](https://github.com/pingcap/tidb/issues/47711) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that an exception might occur when using the `QB_NAME` hint in a prepared statement [#46817](https://github.com/pingcap/tidb/issues/46817) @[jackysp](https://github.com/jackysp) + - Fix the issue of Goroutine leak when using `AUTO_ID_CACHE=1` [#46324](https://github.com/pingcap/tidb/issues/46324) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that TiDB might panic when shutting down [#32110](https://github.com/pingcap/tidb/issues/32110) @[july2993](https://github.com/july2993) + - Fix the issue of not handling locks in the MVCC interface when reading schema diff commit versions from the TiDB schema cache [#48281](https://github.com/pingcap/tidb/issues/48281) @[cfzjywxk](https://github.com/cfzjywxk) + - Fix the issue of duplicate rows in `information_schema.columns` caused by renaming a table [#47064](https://github.com/pingcap/tidb/issues/47064) @[jiyfhust](https://github.com/jiyfhust) + - Fix the bugs in the `LOAD DATA REPLACE INTO` statement [#47995](https://github.com/pingcap/tidb/issues/47995)) @[lance6716](https://github.com/lance6716) + - Fix the issue of `IMPORT INTO` task failure caused by PD leader malfunction for 1 minute [#48307](https://github.com/pingcap/tidb/issues/48307) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue of `ADMIN CHECK` failure caused by creating an index on a date type field [#47426](https://github.com/pingcap/tidb/issues/47426) @[tangenta](https://github.com/tangenta) + - Fix the issue of unsorted row data returned by `TABLESAMPLE` [#48253](https://github.com/pingcap/tidb/issues/48253) @[tangenta](https://github.com/tangenta) + - Fix the TiDB node panic issue that occurs when DDL `jobID` is restored to 0 [#46296](https://github.com/pingcap/tidb/issues/46296) @[jiyfhust](https://github.com/jiyfhust) + ++ TiKV + + - Fix the issue that moving a peer might cause the performance of the Follower Read to deteriorate [#15468](https://github.com/tikv/tikv/issues/15468) @[YuJuncen](https://github.com/YuJuncen) + - Fix the data error of continuously increasing raftstore-applys [#15371](https://github.com/tikv/tikv/issues/15371) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that requests of the TiDB Lightning checksum coprocessor time out when there is online workload [#15565](https://github.com/tikv/tikv/issues/15565) @[lance6716](https://github.com/lance6716) + - Fix security issues by upgrading the version of `lz4-sys` to 1.9.4 [#15621](https://github.com/tikv/tikv/issues/15621) @[SpadeA-Tang](https://github.com/SpadeA-Tang) + - Fix security issues by upgrading the version of `tokio` to 6.5 [#15621](https://github.com/tikv/tikv/issues/15621) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix security issues by removing the `flatbuffer` [#15621](https://github.com/tikv/tikv/issues/15621) @[tonyxuqqi](https://github.com/tonyxuqqi) + - Fix the issue that resolved-ts lag increases when TiKV stores are partitioned [#15679](https://github.com/tikv/tikv/issues/15679) @[hicqu](https://github.com/hicqu) + - Fix the TiKV OOM issue that occurs when restarting TiKV and there are a large number of Raft logs that are not applied [#15770](https://github.com/tikv/tikv/issues/15770) @[overvenus](https://github.com/overvenus) + - Fix the issue that stale peers are retained and block resolved-ts after Regions are merged [#15919](https://github.com/tikv/tikv/issues/15919) @[overvenus](https://github.com/overvenus) + - Fix the issue that the scheduler command variables are incorrect in Grafana on the cloud environment [#15832](https://github.com/tikv/tikv/issues/15832) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that `blob-run-mode` in Titan cannot be updated online [#15978](https://github.com/tikv/tikv/issues/15978) @[tonyxuqqi](https://github.com/tonyxuqqi) + - Fix the issue that TiKV panics due to inconsistent metadata between Regions [#13311](https://github.com/tikv/tikv/issues/13311) @[cfzjywxk](https://github.com/cfzjywxk) + - Fix the issue that TiKV panics when the leader is forced to exit during Online Unsafe Recovery [#15629](https://github.com/tikv/tikv/issues/15629) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that the joint state of DR Auto-Sync might time out when scaling out [#15817](https://github.com/tikv/tikv/issues/15817) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that TiKV coprocessor might return stale data when removing a Raft peer [#16069](https://github.com/tikv/tikv/issues/16069) @[overvenus](https://github.com/overvenus) + - Fix the issue that resolved-ts might be blocked for 2 hours [#39130](https://github.com/pingcap/tidb/issues/39130) @[overvenus](https://github.com/overvenus) + - Fix the issue that Flashback might get stuck when encountering `notLeader` or `regionNotFound` [#15712](https://github.com/tikv/tikv/issues/15712) @[HuSharp](https://github.com/HuSharp) + ++ PD + + - Fix potential security risks of the plugin directory and files [#7094](https://github.com/tikv/pd/issues/7094) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that modified isolation levels are not synchronized to the default placement rules [#7121](https://github.com/tikv/pd/issues/7121) @[rleungx](https://github.com/rleungx) + - Fix the issue that `evict-leader-scheduler` might lose configuration [#6897](https://github.com/tikv/pd/issues/6897) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that the method for counting empty Regions might cause Regions to be unbalanced during the recovery process of BR [#7148](https://github.com/tikv/pd/issues/7148) @[Cabinfever](https://github.com/CabinfeverB) + - Fix the issue that `canSync` and `hasMajority` might be calculated incorrectly for clusters adopting the Data Replication Auto Synchronous (DR Auto-Sync) mode when the configuration of Placement Rules is complex [#7201](https://github.com/tikv/pd/issues/7201) @[disksing](https://github.com/disksing) + - Fix the issue that `available_stores` is calculated incorrectly for clusters adopting the Data Replication Auto Synchronous (DR Auto-Sync) mode [#7221](https://github.com/tikv/pd/issues/7221) @[disksing](https://github.com/disksing) + - Fix the issue that the primary AZ cannot add TiKV nodes when the secondary AZ is down for clusters adopting the Data Replication Auto Synchronous (DR Auto-Sync) mode [#7218](https://github.com/tikv/pd/issues/7218) @[disksing](https://github.com/disksing) + - Fix the issue that adding multiple TiKV nodes to a large cluster might cause TiKV heartbeat reporting to become slow or stuck [#7248](https://github.com/tikv/pd/issues/7248) @[rleungx](https://github.com/rleungx) + - Fix the issue that PD might delete normal Peers when TiKV nodes are unavailable [#7249](https://github.com/tikv/pd/issues/7249) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that it takes a long time to switch the leader in DR Auto-Sync mode [#6988](https://github.com/tikv/pd/issues/6988) @[HuSharp](https://github.com/HuSharp) + - Upgrade the version of Gin Web Framework from v1.8.1 to v1.9.1 to fix some security issues [#7438](https://github.com/tikv/pd/issues/7438) @[niubell](https://github.com/niubell) + ++ TiFlash + + - Fix the issue that the `max_snapshot_lifetime` metric is displayed incorrectly on Grafana [#7713](https://github.com/pingcap/tiflash/issues/7713) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that executing the `ALTER TABLE ... EXCHANGE PARTITION ...` statement causes panic [#8372](https://github.com/pingcap/tiflash/issues/8372) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that the memory usage reported by MemoryTracker is inaccurate [#8128](https://github.com/pingcap/tiflash/issues/8128) @[JinheLin](https://github.com/JinheLin) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that the log backup might get stuck in some scenarios when backing up large wide tables [#15714](https://github.com/tikv/tikv/issues/15714) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that frequent flushes cause log backup to get stuck [#15602](https://github.com/tikv/tikv/issues/15602) @[3pointer](https://github.com/3pointer) + - Fix the issue that the retry after an EC2 metadata connection reset cause degraded backup and restore performance [#47650](https://github.com/pingcap/tidb/issues/47650) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that running PITR multiple times within 1 minute might cause data loss [#15483](https://github.com/tikv/tikv/issues/15483) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that the default values for BR SQL commands and CLI are different, which might cause OOM issues [#48000](https://github.com/pingcap/tidb/issues/48000) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that log backup might panic when the PD owner is transferred [#47533](https://github.com/pingcap/tidb/issues/47533) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that BR generates incorrect URIs for external storage files [#48452](https://github.com/pingcap/tidb/issues/48452) @[3AceShowHand](https://github.com/3AceShowHand) + + + TiCDC + + - Fix the issue that TiCDC server might panic when executing lossy DDL statements in upstream [#9739](https://github.com/pingcap/tiflow/issues/9739) @[hicqu](https://github.com/hicqu) + - Fix the issue that the replication task reports an error when executing `RESUME` with the redo log feature enabled [#9769](https://github.com/pingcap/tiflow/issues/9769) @[hicqu](https://github.com/hicqu) + - Fix the issue that replication lag becomes longer when the TiKV node crashes [#9741](https://github.com/pingcap/tiflow/issues/9741) @[sdojjy](https://github.com/sdojjy) + - Fix the issue that the `WHERE` statement does not use the primary key as the condition when replicating data to TiDB or MySQL [#9988](https://github.com/pingcap/tiflow/issues/9988) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that the workload of a replication task is not distributed evenly across TiCDC nodes [#9839](https://github.com/pingcap/tiflow/issues/9839) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that the interval between replicating DDL statements is too long when redo log is enabled [#9960](https://github.com/pingcap/tiflow/issues/9960) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that the changefeed cannot replicate DML events in bidirectional replication mode if the target table is dropped and then recreated in upstream [#10079](https://github.com/pingcap/tiflow/issues/10079) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that the replication lag becomes longer due to too many NFS files when replicating data to an object storage service [#10041](https://github.com/pingcap/tiflow/issues/10041) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that the TiCDC server might panic when replicating data to an object storage service [#10137](https://github.com/pingcap/tiflow/issues/10137) @[sdojjy](https://github.com/sdojjy) + - Fix the issue that TiCDC accesses the invalid old address during PD scaling up and down [#9584](https://github.com/pingcap/tiflow/issues/9584) @[fubinzh](https://github.com/fubinzh) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that fetching wrong memory information might cause OOM issues in some operating systems [#9762](https://github.com/pingcap/tiflow/issues/9762) @[sdojjy](https://github.com/sdojjy) + + + TiDB Data Migration (DM) + + - Fix the issue that DM skips partition DDLs in optimistic mode [#9788](https://github.com/pingcap/tiflow/issues/9788) @[GMHDBJD](https://github.com/GMHDBJD) + - Fix the issue that DM cannot properly track upstream table schemas when skipping online DDLs [#9587](https://github.com/pingcap/tiflow/issues/9587) @[GMHDBJD](https://github.com/GMHDBJD) + - Fix the issue that replication lag returned by DM keeps growing when a failed DDL is skipped and no subsequent DDLs are executed [#9605](https://github.com/pingcap/tiflow/issues/9605) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that DM skips all DMLs when resuming a task in optimistic mode [#9588](https://github.com/pingcap/tiflow/issues/9588) @[GMHDBJD](https://github.com/GMHDBJD) + + + TiDB Lightning + + - Fix the issue that data import fails when encountering the `write to tikv with no leader returned` error [#45673](https://github.com/pingcap/tidb/issues/45673) @[lance6716](https://github.com/lance6716) + - Fix the issue that data import fails because HTTP retry requests do not use the current request content [#47930](https://github.com/pingcap/tidb/issues/47930) @[lance6716](https://github.com/lance6716) + - Fix the issue that TiDB Lightning gets stuck during `writeToTiKV` [#46321](https://github.com/pingcap/tidb/issues/46321) @[lance6716](https://github.com/lance6716) + - Remove unnecessary `get_regions` calls in physical import mode [#45507](https://github.com/pingcap/tidb/issues/45507) @[mittalrishabh](https://github.com/mittalrishabh) + + + TiDB Binlog + + - Fix the issue that Drainer exits when transporting a transaction greater than 1 GB [#28659](https://github.com/pingcap/tidb/issues/28659) @[jackysp](https://github.com/jackysp) \ No newline at end of file diff --git a/releases/release-6.5.7.md b/releases/release-6.5.7.md new file mode 100644 index 0000000000000..2bbb8f59e2bac --- /dev/null +++ b/releases/release-6.5.7.md @@ -0,0 +1,96 @@ +--- +title: TiDB 6.5.7 Release Notes +summary: Learn about the improvements and bug fixes in TiDB 6.5.7. +--- + +# TiDB 6.5.7 Release Notes + +Release date: January 8, 2024 + +TiDB version: 6.5.7 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) + +## Compatibility changes + ++ Introduce the TiDB configuration item [`performance.force-init-stats`](https://docs.pingcap.com/tidb/v6.5/tidb-configuration-file#force-init-stats-new-in-v657) to control whether TiDB needs to wait for statistics initialization to finish before providing services during TiDB startup [#43385](https://github.com/pingcap/tidb/issues/43385) @[xuyifangreeneyes](https://github.com/xuyifangreeneyes) ++ To reduce the overhead of log printing, TiFlash changes the default value of `logger.level` from `"debug"` to `"info"` [#8568](https://github.com/pingcap/tiflash/issues/8568) @[xzhangxian1008](https://github.com/xzhangxian1008) + +## Improvements + ++ TiDB + + - Optimize memory usage and performance for `ANALYZE` operations on partitioned tables [#47071](https://github.com/pingcap/tidb/issues/47071) [#47104](https://github.com/pingcap/tidb/issues/47104) [#46804](https://github.com/pingcap/tidb/issues/46804) @[hawkingrei](https://github.com/hawkingrei) + - Support Plan Cache to cache execution plans with the `PointGet` operator generated during physical optimization using Optimizer Fix Controls [#44830](https://github.com/pingcap/tidb/issues/44830) @[qw4990](https://github.com/qw4990) + - Enhance the ability to convert `OUTER JOIN` to `INNER JOIN` in specific scenarios [#49616](https://github.com/pingcap/tidb/issues/49616) @[qw4990](https://github.com/qw4990) + ++ TiFlash + + - Reduce the impact of disk performance jitter on read latency [#8583](https://github.com/pingcap/tiflash/issues/8583) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - Improve the table creation performance of the `RESTORE` statement in scenarios with large datasets [#48301](https://github.com/pingcap/tidb/issues/48301) @[Leavrth](https://github.com/Leavrth) + - Resolve compatibility issues between EBS-based snapshot backups and TiDB Lightning imports [#46850](https://github.com/pingcap/tidb/issues/46850) @[YuJuncen](https://github.com/YuJuncen) + - Alleviate the issue that the latency of the PITR log backup progress increases when Region leadership migration occurs [#13638](https://github.com/tikv/tikv/issues/13638) @[YuJuncen](https://github.com/YuJuncen) + + + TiCDC + + - When the downstream is Kafka, the topic expression allows `schema` to be optional and supports specifying a topic name directly [#9763](https://github.com/pingcap/tiflow/issues/9763) @[3AceShowHand](https://github.com/3AceShowHand) + +## Bug fixes + ++ TiDB + + - Fix the issue that TiDB might not simultaneously create the new statistics metadata when a large number of `CREATE TABLE` statements are executed in a short period of time, causing subsequent query estimation to fail to get accurate row count information [#36004](https://github.com/pingcap/tidb/issues/36004) [#38189](https://github.com/pingcap/tidb/issues/38189) @[xuyifangreeneyes](https://github.com/xuyifangreeneyes) + - Fix the issue that TiDB server might consume a significant amount of resources when the enterprise plugin for audit logging is used [#49273](https://github.com/pingcap/tidb/issues/49273) @[lcwangchao](https://github.com/lcwangchao) + - Fix the incorrect error message for `ErrLoadDataInvalidURI` (invalid S3 URI error) [#48164](https://github.com/pingcap/tidb/issues/48164) @[lance6716](https://github.com/lance6716) + - Fix the issue that high CPU usage of TiDB occurs due to long-term memory pressure caused by `tidb_server_memory_limit` [#48741](https://github.com/pingcap/tidb/issues/48741) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that queries containing common table expressions (CTEs) unexpectedly get stuck when the memory limit is exceeded [#49096](https://github.com/pingcap/tidb/issues/49096) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that the same query plan has different `PLAN_DIGEST` values in some cases [#47634](https://github.com/pingcap/tidb/issues/47634) @[King-Dylan](https://github.com/King-Dylan) + - Fix the issue that queries containing CTEs report `runtime error: index out of range [32] with length 32` when `tidb_max_chunk_size` is set to a small value [#48808](https://github.com/pingcap/tidb/issues/48808) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that TiDB server might panic during graceful shutdown [#36793](https://github.com/pingcap/tidb/issues/36793) @[bb7133](https://github.com/bb7133) + - Fix the issue of possible statistics data errors when importing statistics exported from early versions of TiDB [#42931](https://github.com/pingcap/tidb/issues/42931) @[xuyifangreeneyes](https://github.com/xuyifangreeneyes) + - Fix the issue of excessive statistical error in constructing statistics caused by Golang's implicit conversion algorithm [#49801](https://github.com/pingcap/tidb/issues/49801) @[qw4990](https://github.com/qw4990) + - Fix the issue that the optimizer incorrectly converts TiFlash selection path to the DUAL table in specific scenarios [#49285](https://github.com/pingcap/tidb/issues/49285) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that parsing invalid values of `ENUM` or `SET` types would directly cause SQL statement errors [#49487](https://github.com/pingcap/tidb/issues/49487) @[winoros](https://github.com/winoros) + - Fix the issue that `UPDATE` or `DELETE` statements containing `WITH RECURSIVE` CTEs might produce incorrect results [#48969](https://github.com/pingcap/tidb/issues/48969) @[winoros](https://github.com/winoros) + - Fix the issue that using the `_` wildcard in `LIKE` when the data contains trailing spaces can result in incorrect query results [#48983](https://github.com/pingcap/tidb/issues/48983) @[time-and-fate](https://github.com/time-and-fate) + - Fix the issue that a query containing the IndexHashJoin operator gets stuck when memory exceeds `tidb_mem_quota_query` [#49033](https://github.com/pingcap/tidb/issues/49033) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that `LIMIT` and `OPRDERBY` might be invalid in nested `UNION` queries [#49377](https://github.com/pingcap/tidb/issues/49377) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that in non-strict mode (`sql_mode = ''`), truncation during executing `INSERT` still reports an error [#49369](https://github.com/pingcap/tidb/issues/49369) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that TiDB panics and reports an error `invalid memory address or nil pointer dereference` [#42739](https://github.com/pingcap/tidb/issues/42739) @[CbcWestwolf](https://github.com/CbcWestwolf) + - Fix the issue that CTE queries might report an error `type assertion for CTEStorageMap failed` during the retry process [#46522](https://github.com/pingcap/tidb/issues/46522) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that Daylight Saving Time is displayed incorrectly in some time zones [#49586](https://github.com/pingcap/tidb/issues/49586) @[overvenus](https://github.com/overvenus) + - Fix the issue that the completion times of two DDL tasks with dependencies are incorrectly sequenced [#49498](https://github.com/pingcap/tidb/issues/49498) @[tangenta](https://github.com/tangenta) + ++ TiKV + + - Fix the issue that damaged SST files might be spread to other TiKV nodes [#15986](https://github.com/tikv/tikv/issues/15986) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that Resolved TS in stale read might cause TiKV OOM issues when tracking large transactions [#14864](https://github.com/tikv/tikv/issues/14864) @[overvenus](https://github.com/overvenus) + - Fix the issue that TiKV reports the `ServerIsBusy` error because it cannot append the raft log [#15800](https://github.com/tikv/tikv/issues/15800) @[tonyxuqqi](https://github.com/tonyxuqqi) + ++ PD + + - Fix the issue that the `location-labels` set by Placement Rules in SQL are not scheduled as expected under specific conditions [#6637](https://github.com/tikv/pd/issues/6637) @[rleungx](https://github.com/rleungx) + - Fix the issue that the orphan peer is deleted when the number of replicas does not meet the requirements [#7584](https://github.com/tikv/pd/issues/7584) @[bufferflies](https://github.com/bufferflies) + ++ TiFlash + + - Fix the issue that data of TiFlash replicas would still be garbage collected after executing `FLASHBACK DATABASE` [#8450](https://github.com/pingcap/tiflash/issues/8450) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue of memory leak when TiFlash encounters memory limitation during query [#8447](https://github.com/pingcap/tiflash/issues/8447) @[JinheLin](https://github.com/JinheLin) + - Fix incorrect display of maximum percentile time for some panels in Grafana [#8076](https://github.com/pingcap/tiflash/issues/8076) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that the memory usage increases significantly due to slow queries [#8564](https://github.com/pingcap/tiflash/issues/8564) @[JinheLin](https://github.com/JinheLin) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that the log backup task can start but does not work properly if failing to connect to PD during task initialization [#16056](https://github.com/tikv/tikv/issues/16056) @[YuJuncen](https://github.com/YuJuncen) + + + TiCDC + + - Fix the issue that `checkpoint-ts` might get stuck when TiCDC replicates data to downstream MySQL [#10334](https://github.com/pingcap/tiflow/issues/10334) @[zhangjinpeng87](https://github.com/zhangjinpeng87) + - Fix the potential data race issue during `kv-client` initialization [#10095](https://github.com/pingcap/tiflow/issues/10095) @[3AceShowHand](https://github.com/3AceShowHand) diff --git a/releases/release-6.5.8.md b/releases/release-6.5.8.md new file mode 100644 index 0000000000000..872f0d5b79372 --- /dev/null +++ b/releases/release-6.5.8.md @@ -0,0 +1,90 @@ +--- +title: TiDB 6.5.8 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 6.5.8. +--- + +# TiDB 6.5.8 Release Notes + +Release date: February 2, 2024 + +TiDB version: 6.5.8 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) + +## Compatibility changes + +- Introduce the TiKV configuration item [`gc.num-threads`](https://docs.pingcap.com/tidb/v6.5/tikv-configuration-file#num-threads-new-in-v658) to set the number of GC threads when `enable-compaction-filter` is `false` [#16101](https://github.com/tikv/tikv/issues/16101) @[tonyxuqqi](https://github.com/tonyxuqqi) + +## Improvements + ++ TiFlash + + - Reduce the impact of background GC tasks on read and write task latency [#8650](https://github.com/pingcap/tiflash/issues/8650) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + TiCDC + + - Support [querying the downstream synchronization status of a changefeed](https://docs.pingcap.com/tidb/v6.5/ticdc-open-api-v2#query-whether-a-specific-replication-task-is-completed), which helps you determine whether the upstream data changes received by TiCDC have been synchronized to the downstream system completely [#10289](https://github.com/pingcap/tiflow/issues/10289) @[hongyunyan](https://github.com/hongyunyan) + + + TiDB Lightning + + - Improve the performance of `ALTER TABLE` when importing a large number of small tables [#50105](https://github.com/pingcap/tidb/issues/50105) @[D3Hunter](https://github.com/D3Hunter) + +## Bug fixes + ++ TiDB + + - Fix the issue that enforced sorting might become ineffective when a query uses optimizer hints (such as `STREAM_AGG()`) that enforce sorting and its execution plan contains `IndexMerge` [#49605](https://github.com/pingcap/tidb/issues/49605) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that histogram statistics might not be parsed into readable strings when the histogram boundary contains `NULL` [#49823](https://github.com/pingcap/tidb/issues/49823) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that hints cannot be used in `REPLACE INTO` statements [#34325](https://github.com/pingcap/tidb/issues/34325) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that query results are incorrect due to `STREAM_AGG()` incorrectly handling CI [#49902](https://github.com/pingcap/tidb/issues/49902) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that the query result of a range partitioned table is incorrect in some cases due to wrong partition pruning [#50082](https://github.com/pingcap/tidb/issues/50082) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that the auto-increment ID allocation reports an error due to concurrent conflicts when using an auto-increment column with `AUTO_ID_CACHE=1` [#50519](https://github.com/pingcap/tidb/issues/50519) @[tiancaiamao](https://github.com/tiancaiamao) + - Mitigate the issue that TiDB nodes might encounter OOM errors when dealing with a large number of tables or partitions [#50077](https://github.com/pingcap/tidb/issues/50077) @[zimulala](https://github.com/zimulala) + - Fix the issue that data is inconsistent under the TiDB Distributed eXecution Framework (DXF) when executing `ADD INDEX` after the DDL Owner is network isolated [#49773](https://github.com/pingcap/tidb/issues/49773) @[tangenta](https://github.com/tangenta) + - Fix the issue that TiDB might panic when a query contains the Apply operator and the `fatal error: concurrent map writes` error occurs [#50347](https://github.com/pingcap/tidb/issues/50347) @[SeaRise](https://github.com/SeaRise) + - Fix the issue that the `COMMIT` or `ROLLBACK` operation executed through `COM_STMT_EXECUTE` fails to terminate transactions that have timed out [#49151](https://github.com/pingcap/tidb/issues/49151) @[zyguan](https://github.com/zyguan) + - Fix the issue that executing `SELECT INTO OUTFILE` using the `PREPARE` method incorrectly returns a success message instead of an error [#49166](https://github.com/pingcap/tidb/issues/49166) @[qw4990](https://github.com/qw4990) + - Fix the issue that executing `UNIQUE` index lookup with an `ORDER BY` clause might cause an error [#49920](https://github.com/pingcap/tidb/issues/49920) @[jackysp](https://github.com/jackysp) + - Fix the issue that the `DELETE` and `UPDATE` statements using index lookup might report an error when `tidb_multi_statement_mode` mode is enabled [#50012](https://github.com/pingcap/tidb/issues/50012) @[tangenta](https://github.com/tangenta) + - Fix the issue that the `LEADING` hint does not take effect in `UNION ALL` statements [#50067](https://github.com/pingcap/tidb/issues/50067) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that using old interfaces might cause inconsistent metadata for tables [#49751](https://github.com/pingcap/tidb/issues/49751) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that common hints do not take effect in `UNION ALL` statements [#50068](https://github.com/pingcap/tidb/issues/50068) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that TiDB returns wrong query results when processing `ENUM` or `SET` types by constant propagation [#49440](https://github.com/pingcap/tidb/issues/49440) @[winoros](https://github.com/winoros) + ++ TiKV + + - Fix the issue that TiKV might panic when gRPC threads are checking `is_shutdown` [#16236](https://github.com/tikv/tikv/issues/16236) @[pingyu](https://github.com/pingyu) + - Fix the issue that TiKV converts the time zone incorrectly for Brazil and Egypt [#16220](https://github.com/tikv/tikv/issues/16220) @[overvenus](https://github.com/overvenus) + - Fix the issue that TiDB and TiKV might produce inconsistent results when processing `DECIMAL` arithmetic multiplication truncation [#16268](https://github.com/tikv/tikv/issues/16268) @[solotzg](https://github.com/solotzg) + ++ PD + + - Fix the issue that querying a Region without a leader using `pd-ctl` might cause PD to panic [#7630](https://github.com/tikv/pd/issues/7630) @[rleungx](https://github.com/rleungx) + ++ TiFlash + + - Fix the issue that the `lowerUTF8` and `upperUTF8` functions do not allow characters in different cases to occupy different bytes [#8484](https://github.com/pingcap/tiflash/issues/8484) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that TiFlash panics after executing `ALTER TABLE ... MODIFY COLUMN ... NOT NULL`, which changes nullable columns to non-nullable [#8419](https://github.com/pingcap/tiflash/issues/8419) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that after terminating a query, TiFlash crashes due to concurrent data conflicts when a large number of tasks on TiFlash are canceled at the same time [#7432](https://github.com/pingcap/tiflash/issues/7432) @[SeaRise](https://github.com/SeaRise) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that the `Unsupported collation` error is reported when you restore data from backups of an old version [#49466](https://github.com/pingcap/tidb/issues/49466) @[3pointer](https://github.com/3pointer) + - Fix the issue that BR cannot retry when encountering an error while reading file content from S3 [#49942](https://github.com/pingcap/tidb/issues/49942) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that log backup gets stuck after changing the TiKV IP address on the same node [#50445](https://github.com/pingcap/tidb/issues/50445) @[3pointer](https://github.com/3pointer) + + + TiCDC + + - Fix the issue that after filtering out `add table partition` events is configured in `ignore-event`, TiCDC does not replicate other types of DML changes for related partitions to the downstream [#10524](https://github.com/pingcap/tiflow/issues/10524) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that the changefeed reports an error after `TRUNCATE PARTITION` is executed on the upstream table [#10522](https://github.com/pingcap/tiflow/issues/10522) @[sdojjy](https://github.com/sdojjy) + - Fix the issue that the changefeed `resolved ts` does not advance in extreme cases [#10157](https://github.com/pingcap/tiflow/issues/10157) @[sdojjy](https://github.com/sdojjy) + - Fix the issue that TiCDC returns the `ErrChangeFeedAlreadyExists` error when concurrently creating multiple changefeeds [#10430](https://github.com/pingcap/tiflow/issues/10430) @[CharlesCheung96](https://github.com/CharlesCheung96) + + + TiDB Lightning + + - Fix the issue that TiDB Lightning might fail to import data when EBS BR is running [#49517](https://github.com/pingcap/tidb/issues/49517) @[mittalrishabh](https://github.com/mittalrishabh) + - Fix the issue that data might be lost when TiDB Lightning ingests files in batches [#50198](https://github.com/pingcap/tidb/issues/50198) @[D3Hunter](https://github.com/D3Hunter) diff --git a/releases/release-6.5.9.md b/releases/release-6.5.9.md new file mode 100644 index 0000000000000..6e94ff6e38c29 --- /dev/null +++ b/releases/release-6.5.9.md @@ -0,0 +1,137 @@ +--- +title: TiDB 6.5.9 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 6.5.9. +--- + +# TiDB 6.5.9 Release Notes + +Release date: April 12, 2024 + +TiDB version: 6.5.9 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) + +## Compatibility changes + +- Add a TiKV configuration item [`track-and-verify-wals-in-manifest`](https://docs.pingcap.com/tidb/v6.5/tikv-configuration-file#track-and-verify-wals-in-manifest-new-in-v659) for RocksDB, which helps you investigate possible corruption of Write Ahead Log (WAL) [#16549](https://github.com/tikv/tikv/issues/16549) @[v01dstar](https://github.com/v01dstar) +- DR Auto-Sync supports configuring [`wait-recover-timeout`](https://docs.pingcap.com/tidb/v6.5/two-data-centers-in-one-city-deployment#enable-the-dr-auto-sync-mode), which enables you to control the waiting time for switching back to the `sync-recover` status after the network recovers [#6295](https://github.com/tikv/pd/issues/6295) @[disksing](https://github.com/disksing) + +## Improvements + ++ TiDB + + - When `force-init-stats` is set to `true`, TiDB waits for statistics initialization to finish before providing services during TiDB startup. This setting no longer blocks the startup of HTTP servers, which enables users to continue monitoring [#50854](https://github.com/pingcap/tidb/issues/50854) @[hawkingrei](https://github.com/hawkingrei) + - Optimize the issue that the `ANALYZE` statement blocks the metadata lock [#47475](https://github.com/pingcap/tidb/issues/47475) @[wjhuang2016](https://github.com/wjhuang2016) + ++ TiKV + + - Remove unnecessary async blocks to reduce memory usage [#16540](https://github.com/tikv/tikv/issues/16540) @[overvenus](https://github.com/overvenus) + - Avoid performing IO operations on snapshot files in raftstore threads to improve TiKV stability [#16564](https://github.com/tikv/tikv/issues/16564) @[Connor1996](https://github.com/Connor1996) + - Add slow logs for peer and store messages [#16600](https://github.com/tikv/tikv/issues/16600) @[Connor1996](https://github.com/Connor1996) + ++ Tools + + + Backup & Restore (BR) + + - Optimize the Recovery Point Objective (RPO) for log backup during rolling restarts. Now, the checkpoint lag of log backup tasks will be smaller during rolling restarts [#15410](https://github.com/tikv/tikv/issues/15410) @[YuJuncen](https://github.com/YuJuncen) + - Enhance the tolerance of log backup to merge operations. When encountering a reasonably long merge operation, log backup tasks are less likely to enter the error state [#16554](https://github.com/tikv/tikv/issues/16554) @[YuJuncen](https://github.com/YuJuncen) + - Support automatically abandoning log backup tasks when encountering a large checkpoint lag, to avoid prolonged blocking GC and potential cluster issues [#50803](https://github.com/pingcap/tidb/issues/50803) @[RidRisR](https://github.com/RidRisR) + - Alleviate the issue that the latency of the PITR log backup progress increases when Region leadership migration occurs [#13638](https://github.com/tikv/tikv/issues/13638) @[YuJuncen](https://github.com/YuJuncen) + - Improve the speed of merging SST files during data restore by using a more efficient algorithm [#50613](https://github.com/pingcap/tidb/issues/50613) @[Leavrth](https://github.com/Leavrth) + - Support ingesting SST files in batch during data restore [#16267](https://github.com/tikv/tikv/issues/16267) @[3pointer](https://github.com/3pointer) + - Remove an outdated compatibility check when using Google Cloud Storage (GCS) as the external storage [#50533](https://github.com/pingcap/tidb/issues/50533) @[lance6716](https://github.com/lance6716) + - Print the information of the slowest Region that affects global checkpoint advancement in logs and metrics during log backups [#51046](https://github.com/pingcap/tidb/issues/51046) @[YuJuncen](https://github.com/YuJuncen) + - Refactor the BR exception handling mechanism to increase tolerance for unknown errors [#47656](https://github.com/pingcap/tidb/issues/47656) @[3pointer](https://github.com/3pointer) + +## Bug fixes + ++ TiDB + + - Fix the issue that dropped tables are still counted by the Grafana `Stats Healthy Distribution` panel [#39349](https://github.com/pingcap/tidb/issues/39349) @[xuyifangreeneyes](https://github.com/xuyifangreeneyes) + - Fix the issue that TiDB does not handle the `WHERE ` filtering condition in a SQL statement when the query of that statement involves the `MemTableScan` operator [#40937](https://github.com/pingcap/tidb/issues/40937) @[zhongzc](https://github.com/zhongzc) + - Fix the issue that query results might be incorrect when the `HAVING` clause in a subquery contains correlated columns [#51107](https://github.com/pingcap/tidb/issues/51107) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that query results might be incorrect when you use Common Table Expressions (CTE) to access partitioned tables with missing statistics [#51873](https://github.com/pingcap/tidb/issues/51873) @[qw4990](https://github.com/qw4990) + - Fix the issue that query execution using MPP might lead to incorrect query results when a SQL statement contains `JOIN` and the `SELECT` list in the statement contains only constants [#50358](https://github.com/pingcap/tidb/issues/50358) @[yibin87](https://github.com/yibin87) + - Fix the issue that the `AUTO_INCREMENT` attribute causes non-consecutive IDs due to unnecessary transaction conflicts when assigning auto-increment IDs [#50819](https://github.com/pingcap/tidb/issues/50819) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the monitoring metric `tidb_statistics_auto_analyze_total` on Grafana is not displayed as an integer [#51051](https://github.com/pingcap/tidb/issues/51051) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that errors might be returned during the concurrent merging of global statistics for partitioned tables [#48713](https://github.com/pingcap/tidb/issues/48713) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that getting the default value of a column returns an error if the column default value is dropped [#50043](https://github.com/pingcap/tidb/issues/50043) [#51324](https://github.com/pingcap/tidb/issues/51324) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that the `INSERT ignore` statement cannot fill in default values when the column is write-only [#40192](https://github.com/pingcap/tidb/issues/40192) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that TiDB crashes when `shuffleExec` quits unexpectedly [#48230](https://github.com/pingcap/tidb/issues/48230) @[wshwsh12](https://github.com/wshwsh12) + - Fix the goroutine leak issue that might occur when the `HashJoin` operator fails to spill to disk [#50841](https://github.com/pingcap/tidb/issues/50841) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that renaming tables does not take effect when committing multiple statements in a transaction [#39664](https://github.com/pingcap/tidb/issues/39664) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the query result is incorrect when the `IN()` predicate contains `NULL` [#51560](https://github.com/pingcap/tidb/issues/51560) @[winoros](https://github.com/winoros) + - Fix the issue that querying JSON of `BINARY` type might cause an error in some cases [#51547](https://github.com/pingcap/tidb/issues/51547) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that parallel `Apply` might generate incorrect results when the table has a clustered index [#51372](https://github.com/pingcap/tidb/issues/51372) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that the `init-stats` process might cause TiDB to panic and the `load stats` process to quit [#51581](https://github.com/pingcap/tidb/issues/51581) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `tidb_merge_partition_stats_concurrency` variable does not take effect when `auto analyze` is processing partitioned tables [#47594](https://github.com/pingcap/tidb/issues/47594) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that after the time window for automatic statistics updates is configured, statistics might still be updated outside that time window [#49552](https://github.com/pingcap/tidb/issues/49552) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `approx_percentile` function might cause TiDB panic [#40463](https://github.com/pingcap/tidb/issues/40463) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that `BIT` type columns might cause query errors due to decode failures when they are involved in calculations of some functions [#49566](https://github.com/pingcap/tidb/issues/49566) [#50850](https://github.com/pingcap/tidb/issues/50850) [#50855](https://github.com/pingcap/tidb/issues/50855) @[jiyfhust](https://github.com/jiyfhust) + - Fix the goroutine leak issue that occurs when the memory usage of CTE queries exceed limits [#50337](https://github.com/pingcap/tidb/issues/50337) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that TiDB does not listen to the corresponding port when `force-init-stats` is configured [#51473](https://github.com/pingcap/tidb/issues/51473) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that `ALTER TABLE ... COMPACT TIFLASH REPLICA` might incorrectly end when the primary key type is `VARCHAR` [#51810](https://github.com/pingcap/tidb/issues/51810) @[breezewish](https://github.com/breezewish) + - Fix the issue that the `tidb_gogc_tuner_threshold` system variable is not adjusted accordingly after the `tidb_server_memory_limit` variable is modified [#48180](https://github.com/pingcap/tidb/issues/48180) @[hawkingrei](https://github.com/hawkingrei) + - Fix the `Can't find column ...` error that might occur when aggregate functions are used for group calculations [#50926](https://github.com/pingcap/tidb/issues/50926) @[qw4990](https://github.com/qw4990) + - Fix the issue that the `REVERSE` function reports an error when processing columns of the `BIT` type [#50850](https://github.com/pingcap/tidb/issues/50850) [#50855](https://github.com/pingcap/tidb/issues/50855) @[jiyfhust](https://github.com/jiyfhust) + - Fix the issue that `INSERT IGNORE` reports an error when inserting data in bulk into a table undergoing a DDL operation [#50993](https://github.com/pingcap/tidb/issues/50993) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that the TiDB server adds a label via the HTTP interface and returns success, but does not take effect [#51427](https://github.com/pingcap/tidb/issues/51427) @[you06](https://github.com/you06) + - Fix the issue that the type returned by the `IFNULL` function is inconsistent with MySQL [#51765](https://github.com/pingcap/tidb/issues/51765) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that the TiDB server is marked as health before the initialization is complete [#51596](https://github.com/pingcap/tidb/issues/51596) @[shenqidebaozi](https://github.com/shenqidebaozi) + - Fix the issue that querying the `TIDB_HOT_REGIONS` table might incorrectly return `INFORMATION_SCHEMA` tables [#50810](https://github.com/pingcap/tidb/issues/50810) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that `EXCHANGE PARTITION` incorrectly processes foreign keys [#51807](https://github.com/pingcap/tidb/issues/51807) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that executing queries containing CTEs causes TiDB to panic [#41688](https://github.com/pingcap/tidb/issues/41688) @[srstack](https://github.com/srstack) + ++ TiKV + + - Fix the issue that after the process of destroying Peer is interrupted by applying snapshots, it does not resume even after applying snapshots is complete [#16561](https://github.com/tikv/tikv/issues/16561) @[tonyxuqqi](https://github.com/tonyxuqqi) + - Fix the issue that inactive Write Ahead Logs (WALs) in RocksDB might corrupt data [#16705](https://github.com/tikv/tikv/issues/16705) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that TiKV converts the time zone incorrectly for Brazil and Egypt [#16220](https://github.com/tikv/tikv/issues/16220) @[overvenus](https://github.com/overvenus) + - Fix the issue that the monitoring metric `tikv_unified_read_pool_thread_count` has no data in some cases [#16629](https://github.com/tikv/tikv/issues/16629) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that JSON integers greater than the maximum `INT64` value but less than the maximum `UINT64` value are parsed as `FLOAT64` by TiKV, resulting in inconsistency with TiDB [#16512](https://github.com/tikv/tikv/issues/16512) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that during the execution of an optimistic transaction, if other transactions initiate the resolving lock operation on it, there is a small chance that the atomicity of the transaction might be broken if the transaction's primary key has data that was previously committed in Async Commit or 1PC mode [#16620](https://github.com/tikv/tikv/issues/16620) @[MyonKeminta](https://github.com/MyonKeminta) + ++ PD + + - Fix the issue that the scaling progress is not correctly displayed [#7726](https://github.com/tikv/pd/issues/7726) @[CabinfeverB](https://github.com/CabinfeverB) + - Fix the issue that data race occurs when the `MergeLabels` function is called [#7535](https://github.com/tikv/pd/issues/7535) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that the PD monitoring item `learner-peer-count` does not synchronize the old value after a leader switch [#7728](https://github.com/tikv/pd/issues/7728) @[CabinfeverB](https://github.com/CabinfeverB) + - Fix the issue that the query result of `SHOW CONFIG` includes the deprecated configuration item `trace-region-flow` [#7917](https://github.com/tikv/pd/issues/7917) @[rleungx](https://github.com/rleungx) + ++ TiFlash + + - Fix the issue that TiFlash might panic due to unstable network connections with PD during replica migration [#8323](https://github.com/pingcap/tiflash/issues/8323) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash might crash due to data race in case of remote reads [#8685](https://github.com/pingcap/tiflash/issues/8685) @[solotzg](https://github.com/solotzg) + - Fix the issue that the `ENUM` column might cause TiFlash to crash during chunk encoding [#8674](https://github.com/pingcap/tiflash/issues/8674) @[yibin87](https://github.com/yibin87) + - Fix the issue that TiFlash might panic when you insert data to columns with invalid default values in non-strict `sql_mode` [#8803](https://github.com/pingcap/tiflash/issues/8803) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that if Region migration, split, or merge occurs after the precision of a `TIME` column is modified, queries might fail [#8601](https://github.com/pingcap/tiflash/issues/8601) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that too many logs are printed when a full backup fails [#51572](https://github.com/pingcap/tidb/issues/51572) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that removing a log backup task after it is paused does not immediately restore the GC safepoint [#52082](https://github.com/pingcap/tidb/issues/52082) @[3pointer](https://github.com/3pointer) + - Fix the issue that BR could not back up the `AUTO_RANDOM` ID allocation progress in a union clustered index that contains an `AUTO_RANDOM` column [#52255](https://github.com/pingcap/tidb/issues/52255) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that stopping a log backup task causes TiDB to crash [#50839](https://github.com/pingcap/tidb/issues/50839) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that TiKV panics when a full backup fails to find a peer in some extreme cases [#16394](https://github.com/tikv/tikv/issues/16394) @[Leavrth](https://github.com/Leavrth) + + + TiCDC + + - Fix the issue that `snapshot lost caused by GC` is not reported in time when resuming a changefeed and the `checkpoint-ts` of the changefeed is smaller than the GC safepoint of TiDB [#10463](https://github.com/pingcap/tiflow/issues/10463) @[sdojjy](https://github.com/sdojjy) + - Fix the issue that data is written to a wrong CSV file due to wrong BarrierTS in scenarios where DDL statements are executed frequently [#10668](https://github.com/pingcap/tiflow/issues/10668) @[lidezhu](https://github.com/lidezhu) + - Fix the issue that the Syncpoint table might be incorrectly replicated [#10576](https://github.com/pingcap/tiflow/issues/10576) @[asddongmen](https://github.com/asddongmen) + - Fix the issue TiCDC panics when scheduling table replication tasks [#10613](https://github.com/pingcap/tiflow/issues/10613) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that data race in the KV client causes TiCDC to panic [#10718](https://github.com/pingcap/tiflow/issues/10718) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that the file sequence number generated by the storage service might not increment correctly when using the storage sink [#10352](https://github.com/pingcap/tiflow/issues/10352) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that TiCDC cannot access Azure and GCS properly in storage sink scenarios [#10592](https://github.com/pingcap/tiflow/issues/10592) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that the old value part of `open-protocol` incorrectly outputs the default value according to the `STRING` type instead of its actual type [#10803](https://github.com/pingcap/tiflow/issues/10803) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that a changefeed with eventual consistency enabled might fail when the object storage sink encounters a temporary failure [#10710](https://github.com/pingcap/tiflow/issues/10710) @[CharlesCheung96](https://github.com/CharlesCheung96) + + + TiDB Data Migration (DM) + + - Fix the issue that data is lost when the upstream primary key is of binary type [#10672](https://github.com/pingcap/tiflow/issues/10672) @[GMHDBJD](https://github.com/GMHDBJD) + + + TiDB Lightning + + - Fix the issue that TiDB Lightning reports an error when encountering invalid symbolic link files during file scanning [#49423](https://github.com/pingcap/tidb/issues/49423) @[lance6716](https://github.com/lance6716) \ No newline at end of file diff --git a/releases/release-6.6.0.md b/releases/release-6.6.0.md index dac4d3688ecc5..4507d5a09ca21 100644 --- a/releases/release-6.6.0.md +++ b/releases/release-6.6.0.md @@ -13,7 +13,7 @@ TiDB version: 6.6.0-[DMR](/releases/versioning.md#development-milestone-releases > > The TiDB 6.6.0-DMR documentation has been [archived](https://docs-archive.pingcap.com/tidb/v6.6/). PingCAP encourages you to use [the latest LTS version](https://docs.pingcap.com/tidb/stable) of the TiDB database. -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.6/quick-start-with-tidb) | [Installation package](https://www.pingcap.com/download/?version=v6.6.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.6/quick-start-with-tidb) In v6.6.0-DMR, the key new features and improvements are as follows: @@ -168,11 +168,11 @@ In v6.6.0-DMR, the key new features and improvements are as follows: For more information, see [documentation](/placement-rules-in-sql.md#specify-survival-preferences). -* Support rolling back DDL operations via the `FLASHBACK CLUSTER TO TIMESTAMP` statement [#14088](https://github.com/tikv/tikv/pull/14088) @[Defined2014](https://github.com/Defined2014) @[JmPotato](https://github.com/JmPotato) +* Support rolling back DDL operations via the `FLASHBACK CLUSTER TO TIMESTAMP` statement [#14045](https://github.com/tikv/tikv/issues/14045) @[Defined2014](https://github.com/Defined2014) @[JmPotato](https://github.com/JmPotato) - The [`FLASHBACK CLUSTER TO TIMESTAMP`](/sql-statements/sql-statement-flashback-to-timestamp.md) statement supports restoring the entire cluster to a specified point in time within the Garbage Collection (GC) lifetime. In TiDB v6.6.0, this feature adds support for rolling back DDL operations. This can be used to quickly undo a DML or DDL misoperation on a cluster, roll back a cluster within minutes, and roll back a cluster multiple times on the timeline to determine when specific data changes occurred. + The [`FLASHBACK CLUSTER TO TIMESTAMP`](/sql-statements/sql-statement-flashback-cluster.md) statement supports restoring the entire cluster to a specified point in time within the Garbage Collection (GC) lifetime. In TiDB v6.6.0, this feature adds support for rolling back DDL operations. This can be used to quickly undo a DML or DDL misoperation on a cluster, roll back a cluster within minutes, and roll back a cluster multiple times on the timeline to determine when specific data changes occurred. - For more information, see [documentation](/sql-statements/sql-statement-flashback-to-timestamp.md). + For more information, see [documentation](/sql-statements/sql-statement-flashback-cluster.md). ### SQL @@ -318,7 +318,7 @@ In v6.6.0-DMR, the key new features and improvements are as follows: For more information, see [documentation](/enable-tls-between-components.md). -* TiDB Lightning supports accessing Amazon S3 data via AWS IAM role keys and session tokens [#4075](https://github.com/pingcap/tidb/issues/40750) @[okJiang](https://github.com/okJiang) +* TiDB Lightning supports accessing Amazon S3 data via AWS IAM role keys and session tokens [#40750](https://github.com/pingcap/tidb/issues/40750) @[okJiang](https://github.com/okJiang) Before v6.6.0, TiDB Lightning only supports accessing S3 data via AWS IAM **user's access keys** (each access key consists of an access key ID and a secret access key) so you cannot use a temporary session token to access S3 data. Starting from v6.6.0, TiDB Lightning supports accessing S3 data via AWS IAM **role's access keys + session tokens** as well to improve the data security. @@ -356,6 +356,7 @@ In v6.6.0-DMR, the key new features and improvements are as follows: | [`tidb_enable_foreign_key`](/system-variables.md#tidb_enable_foreign_key-new-in-v630) | Modified | This variable controls whether to enable the foreign key feature. The default value changes from `OFF` to `ON`, which means enabling foreign key by default. | | `tidb_enable_general_plan_cache` | Modified | This variable controls whether to enable General Plan Cache. Starting from v6.6.0, this variable is renamed to [`tidb_enable_non_prepared_plan_cache`](/system-variables.md#tidb_enable_non_prepared_plan_cache). | | [`tidb_enable_historical_stats`](/system-variables.md#tidb_enable_historical_stats) | Modified | This variable controls whether to enable historical statistics. The default value changes from `OFF` to `ON`, which means that historical statistics are enabled by default. | +| [`tidb_enable_plan_replayer_capture`](/system-variables.md#tidb_enable_plan_replayer_capture) | Modified | This variable takes effect starting from v6.6.0 and controls whether to enable the [`PLAN REPLAYER CAPTURE` feature](/sql-plan-replayer.md#use-plan-replayer-capture-to-capture-target-plans). The default value changes from `OFF` to `ON`, which means that the `PLAN REPLAYER CAPTURE` feature is enabled by default. | | [`tidb_enable_telemetry`](/system-variables.md#tidb_enable_telemetry-new-in-v402) | Modified | The default value changes from `ON` to `OFF`, which means that telemetry is disabled by default in TiDB. | | `tidb_general_plan_cache_size` | Modified | This variable controls the maximum number of execution plans that can be cached by General Plan Cache. Starting from v6.6.0, this variable is renamed to [`tidb_non_prepared_plan_cache_size`](/system-variables.md#tidb_non_prepared_plan_cache_size). | | [`tidb_replica_read`](/system-variables.md#tidb_replica_read-new-in-v40) | Modified | A new value option `learner` is added for this variable to specify the learner replicas with which TiDB reads data from read-only nodes. | @@ -366,7 +367,6 @@ In v6.6.0-DMR, the key new features and improvements are as follows: | [`tidb_ddl_distribute_reorg`](https://docs.pingcap.com/tidb/v6.6/system-variables#tidb_ddl_distribute_reorg-new-in-v660) | Newly added | This variable controls whether to enable distributed execution of the DDL reorg phase to accelerate this phase. The default value `OFF` means not to enable distributed execution of the DDL reorg phase by default. Currently, this variable takes effect only for `ADD INDEX`. | | [`tidb_enable_historical_stats_for_capture`](/system-variables.md#tidb_enable_historical_stats_for_capture) | Newly added | This variable controls whether the information captured by `PLAN REPLAYER CAPTURE` includes historical statistics by default. The default value `OFF` means that historical statistics are not included by default. | | [`tidb_enable_plan_cache_for_param_limit`](/system-variables.md#tidb_enable_plan_cache_for_param_limit-new-in-v660) | Newly added | This variable controls whether Prepared Plan Cache caches execution plans that contain `COUNT` after `Limit`. The default value is `ON`, which means Prepared Plan Cache supports caching such execution plans. Note that Prepared Plan Cache does not support caching execution plans with a `COUNT` condition that counts a number greater than 10000. | -| [`tidb_enable_plan_replayer_capture`](/system-variables.md#tidb_enable_plan_replayer_capture) | Newly added | This variable controls whether to enable the [`PLAN REPLAYER CAPTURE` feature](/sql-plan-replayer.md#use-plan-replayer-capture-to-capture-target-plans). The default value `OFF` means to disable the `PLAN REPLAYER CAPTURE` feature. | | [`tidb_enable_resource_control`](/system-variables.md#tidb_enable_resource_control-new-in-v660) | Newly added | This variable controls whether to enable the resource control feature. The default value is `OFF`. When this variable is set to `ON`, the TiDB cluster supports resource isolation of applications based on resource groups. | | [`tidb_historical_stats_duration`](/system-variables.md#tidb_historical_stats_duration-new-in-v660) | Newly added | This variable controls how long the historical statistics are retained in storage. The default value is 7 days. | | [`tidb_index_join_double_read_penalty_cost_rate`](/system-variables.md#tidb_index_join_double_read_penalty_cost_rate-new-in-v660) | Newly added | This variable controls whether to add some penalty cost to the selection of index join. The default value `0` means that this feature is disabled by default. | @@ -471,7 +471,7 @@ In v6.6.0-DMR, the key new features and improvements are as follows: + TiCDC - Support batch `UPDATE` DML statements to improve TiCDC replication performance [#8084](https://github.com/pingcap/tiflow/issues/8084) @[amyangfei](https://github.com/amyangfei) - - Implement MQ sink and MySQL sink in the asynchronous mode to improve the sink throughput [#5928](https://github.com/pingcap/tiflow/issues/5928) @[hicqu](https://github.com/hicqu) @[hi-rustin](https://github.com/hi-rustin) + - Implement MQ sink and MySQL sink in the asynchronous mode to improve the sink throughput [#5928](https://github.com/pingcap/tiflow/issues/5928) @[hicqu](https://github.com/hicqu) @[hi-rustin](https://github.com/Rustin170506) + TiDB Data Migration (DM) @@ -518,7 +518,7 @@ In v6.6.0-DMR, the key new features and improvements are as follows: - Fix the issue that resources are not released when disabling the resource management module [#40546](https://github.com/pingcap/tidb/issues/40546) @[zimulala](https://github.com/zimulala) - Fix the issue that TTL tasks cannot trigger statistics updates in time [#40109](https://github.com/pingcap/tidb/issues/40109) @[YangKeao](https://github.com/YangKeao) - Fix the issue that unexpected data is read because TiDB improperly handles `NULL` values when constructing key ranges [#40158](https://github.com/pingcap/tidb/issues/40158) @[tiancaiamao](https://github.com/tiancaiamao) - - Fix the issue that illegal values are written to a table when the `MODIFT COLUMN` statement also changes the default value of a column [#40164](https://github.com/pingcap/tidb/issues/40164) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that illegal values are written to a table when the `MODIFY COLUMN` statement also changes the default value of a column [#40164](https://github.com/pingcap/tidb/issues/40164) @[wjhuang2016](https://github.com/wjhuang2016) - Fix the issue that the adding index operation is inefficient due to invalid Region cache when there are many Regions in a table [#38436](https://github.com/pingcap/tidb/issues/38436) @[tangenta](https://github.com/tangenta) - Fix data race occurred in allocating auto-increment IDs [#40584](https://github.com/pingcap/tidb/issues/40584) @[Dousir9](https://github.com/Dousir9) - Fix the issue that the implementation of the not operator in JSON is incompatible with the implementation in MySQL [#40683](https://github.com/pingcap/tidb/issues/40683) @[YangKeao](https://github.com/YangKeao) @@ -588,7 +588,7 @@ In v6.6.0-DMR, the key new features and improvements are as follows: - Fix the issue of insufficient duration that redo log can tolerate for S3 storage failure [#8089](https://github.com/pingcap/tiflow/issues/8089) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the issue that changefeed might get stuck in special scenarios such as when scaling in or scaling out TiKV or TiCDC nodes [#8174](https://github.com/pingcap/tiflow/issues/8174) @[hicqu](https://github.com/hicqu) - Fix the issue of too high traffic among TiKV nodes [#14092](https://github.com/tikv/tikv/issues/14092) @[overvenus](https://github.com/overvenus) - - Fix the performance issues of TiCDC in terms of CPU usage, memory control, and throughput when the pull-based sink is enabled [#8142](https://github.com/pingcap/tiflow/issues/8142) [#8157](https://github.com/pingcap/tiflow/issues/8157) [#8001](https://github.com/pingcap/tiflow/issues/8001) [#5928](https://github.com/pingcap/tiflow/issues/5928) @[hicqu](https://github.com/hicqu) @[hi-rustin](https://github.com/hi-rustin) + - Fix the performance issues of TiCDC in terms of CPU usage, memory control, and throughput when the pull-based sink is enabled [#8142](https://github.com/pingcap/tiflow/issues/8142) [#8157](https://github.com/pingcap/tiflow/issues/8157) [#8001](https://github.com/pingcap/tiflow/issues/8001) [#5928](https://github.com/pingcap/tiflow/issues/5928) @[hicqu](https://github.com/hicqu) @[hi-rustin](https://github.com/Rustin170506) + TiDB Data Migration (DM) diff --git a/releases/release-7.0.0.md b/releases/release-7.0.0.md index a420208f94541..8900eb5d9e030 100644 --- a/releases/release-7.0.0.md +++ b/releases/release-7.0.0.md @@ -9,7 +9,7 @@ Release date: March 30, 2023 TiDB version: 7.0.0-[DMR](/releases/versioning.md#development-milestone-releases) -Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.0/quick-start-with-tidb) | [Installation package](https://www.pingcap.com/download/?version=v7.0.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.0/quick-start-with-tidb) In v7.0.0-DMR, the key new features and improvements are as follows: @@ -86,7 +86,7 @@ In v7.0.0-DMR, the key new features and improvements are as follows: Starting from TiDB v7.0.0, Fast Online DDL and PITR are fully compatible. When restoring cluster data through PITR, the index operations added via Fast Online DDL during log backup will be automatically replayed to achieve compatibility. - For more information, see [documentation](/ddl-introduction.md). + For more information, see [documentation](/best-practices/ddl-introduction.md). * TiFlash supports null-aware semi join and null-aware anti semi join operators [#6674](https://github.com/pingcap/tiflash/issues/6674) @[gengliqi](https://github.com/gengliqi) @@ -142,7 +142,7 @@ In v7.0.0-DMR, the key new features and improvements are as follows: TiDB v6.5.0 supports creating ordinary secondary indexes via Fast Online DDL. TiDB v7.0.0 supports creating unique indexes via Fast Online DDL. Compared to v6.1.0, adding unique indexes to large tables is expected to be several times faster with improved performance. - For more information, see [documentation](/ddl-introduction.md). + For more information, see [documentation](/best-practices/ddl-introduction.md). ### Reliability @@ -168,7 +168,7 @@ In v7.0.0-DMR, the key new features and improvements are as follows: TiDB v7.0.0 introduces a checkpoint mechanism for [Fast Online DDL](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630), which significantly improves its fault tolerance and automatic recovery capabilities. By periodically recording and synchronizing the DDL progress, ongoing DDL operations can continue to be executed in Fast Online DDL mode even if there is a TiDB DDL Owner failure or switch. This makes the execution of DDL more stable and efficient. - For more information, see [documentation](/ddl-introduction.md). + For more information, see [documentation](/best-practices/ddl-introduction.md). * TiFlash supports spilling to disk [#6528](https://github.com/pingcap/tiflash/issues/6528) @[windtalker](https://github.com/windtalker) @@ -248,7 +248,7 @@ In v7.0.0-DMR, the key new features and improvements are as follows: * [DBeaver](https://dbeaver.io/) v23.0.1 supports TiDB by default [#17396](https://github.com/dbeaver/dbeaver/issues/17396) @[Icemap](https://github.com/Icemap) - Provides an independent TiDB module, icon, and logo. - - The default configuration supports [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless), making it easier to connect to TiDB Serverless. + - The default configuration supports [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter), making it easier to connect to {{{ .starter }}}. - Supports identifying TiDB versions to display or hide foreign key tabs. - Supports visualizing SQL execution plans in `EXPLAIN` results. - Supports highlighting TiDB keywords such as `PESSIMISTIC`, `OPTIMISTIC`, `AUTO_RANDOM`, `PLACEMENT`, `POLICY`, `REORGANIZE`, `EXCHANGE`, `CACHE`, `NONCLUSTERED`, and `CLUSTERED`. @@ -268,7 +268,7 @@ In v7.0.0-DMR, the key new features and improvements are as follows: For more information, see [documentation](/sql-statements/sql-statement-load-data.md). -* TiDB Lightning supports enabling compressed transfers when sending key-value pairs to TiKV (GA) [#41163](https://github.com/pingcap/tidb/issues/41163) @[gozssky](https://github.com/gozssky) +* TiDB Lightning supports enabling compressed transfers when sending key-value pairs to TiKV (GA) [#41163](https://github.com/pingcap/tidb/issues/41163) @[sleepymole](https://github.com/sleepymole) Starting from v6.6.0, TiDB Lightning supports compressing locally encoded and sorted key-value pairs for network transfer when sending them to TiKV, thus reducing the amount of data transferred over the network and lowering the network bandwidth overhead. In the earlier TiDB versions before this feature is supported, TiDB Lightning requires relatively high network bandwidth and incurs high traffic charges in case of large data volumes. @@ -345,6 +345,7 @@ In v7.0.0-DMR, the key new features and improvements are as follows: | [`tidb_opt_enable_late_materialization`](/system-variables.md#tidb_opt_enable_late_materialization-new-in-v700) | Newly added | This variable controls whether to enable the [TiFlash Late Materialization](/tiflash/tiflash-late-materialization.md) feature. The default value is `OFF`, which means the feature is not enabled. | | [`tidb_opt_ordering_index_selectivity_threshold`](/system-variables.md#tidb_opt_ordering_index_selectivity_threshold-new-in-v700) | Newly added | This variable controls how the optimizer selects indexes when the SQL statement contains `ORDER BY` and `LIMIT` clauses and has filtering conditions. | | [`tidb_pessimistic_txn_fair_locking`](/system-variables.md#tidb_pessimistic_txn_fair_locking-new-in-v700) | Newly added | Controls whether to enable the enhanced pessimistic lock-waking model to reduce the tail latency of transactions under single-row conflict scenarios. The default value is `ON`. When the cluster is upgraded from an earlier version to v7.0.0 or later, the value of this variable is set to `OFF`. | +| [`tidb_slow_txn_log_threshold`](/system-variables.md#tidb_slow_txn_log_threshold-new-in-v700) | Newly added | Sets the threshold for slow transaction logging. When the execution time of a transaction exceeds this threshold, TiDB logs detailed information about the transaction. The default value `0` means that this feature is disabled. | | [`tidb_ttl_running_tasks`](/system-variables.md#tidb_ttl_running_tasks-new-in-v700) | Newly added | This variable is used to limit the concurrency of TTL tasks across the entire cluster. The default value `-1` means that the TTL tasks are the same as the number of TiKV nodes. | ### Configuration file parameters @@ -411,15 +412,15 @@ In v7.0.0-DMR, the key new features and improvements are as follows: - Support splitting transactions in the redo applier to improve its throughput and reduce RTO in disaster recovery scenarios [#8318](https://github.com/pingcap/tiflow/issues/8318) @[CharlesCheung96](https://github.com/CharlesCheung96) - Improve the table scheduling to split a single table more evenly across various TiCDC nodes [#8247](https://github.com/pingcap/tiflow/issues/8247) @[overvenus](https://github.com/overvenus) - - Add the Large Row monitoring metrics in MQ sink [#8286](https://github.com/pingcap/tiflow/issues/8286) @[hi-rustin](https://github.com/hi-rustin) + - Add the Large Row monitoring metrics in MQ sink [#8286](https://github.com/pingcap/tiflow/issues/8286) @[hi-rustin](https://github.com/Rustin170506) - Reduce network traffic between TiKV and TiCDC nodes in scenarios where a Region contains data of multiple tables [#6346](https://github.com/pingcap/tiflow/issues/6346) @[overvenus](https://github.com/overvenus) - - Move the P99 metrics panel of Checkpoint TS and Resolved TS to the Lag analyze panel [#8524](https://github.com/pingcap/tiflow/issues/8524) @[hi-rustin](https://github.com/hi-rustin) + - Move the P99 metrics panel of Checkpoint TS and Resolved TS to the Lag analyze panel [#8524](https://github.com/pingcap/tiflow/issues/8524) @[hi-rustin](https://github.com/Rustin170506) - Support applying DDL events in redo logs [#8361](https://github.com/pingcap/tiflow/issues/8361) @[CharlesCheung96](https://github.com/CharlesCheung96) - Support splitting and scheduling tables to TiCDC nodes based on upstream write throughput [#7720](https://github.com/pingcap/tiflow/issues/7720) @[overvenus](https://github.com/overvenus) + TiDB Lightning - - TiDB Lightning Physical Import Mode supports separating data import and index import to improve import speed and stability [#42132](https://github.com/pingcap/tidb/issues/42132) @[gozssky](https://github.com/gozssky) + - TiDB Lightning Physical Import Mode supports separating data import and index import to improve import speed and stability [#42132](https://github.com/pingcap/tidb/issues/42132) @[sleepymole](https://github.com/sleepymole) Add the `add-index-by-sql` parameter. The default value is `false`, which means that TiDB Lightning encodes both row data and index data into KV pairs and import them into TiKV together. If you set it to `true`, it means that TiDB Lightning adds indexes via the `ADD INDEX` SQL statement after importing the row data to improve import speed and stability. @@ -504,7 +505,7 @@ We would like to thank the following contributors from the TiDB community: - [BornChanger](https://github.com/BornChanger) - [Dousir9](https://github.com/Dousir9) - [erwadba](https://github.com/erwadba) -- [HappyUncle](https://github.com/HappyUncle) +- [happy-v587](https://github.com/happy-v587) - [jiyfhust](https://github.com/jiyfhust) - [L-maple](https://github.com/L-maple) - [liumengya94](https://github.com/liumengya94) diff --git a/releases/release-7.1.0.md b/releases/release-7.1.0.md index bd9a1e60fc309..ffb18261b535e 100644 --- a/releases/release-7.1.0.md +++ b/releases/release-7.1.0.md @@ -9,7 +9,7 @@ Release date: May 31, 2023 TiDB version: 7.1.0 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v7.1.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) TiDB 7.1.0 is a Long-Term Support Release (LTS). @@ -74,7 +74,7 @@ Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, impr TiDB supports LDAP authentication, which is compatible with MySQL 8.0. - Audit log enhancement (Enterprise Edition only) + Audit log enhancement (Enterprise Edition only) TiDB Enterprise Edition enhances the database auditing feature. It significantly improves the system auditing capacity by providing more fine-grained event filtering controls, more user-friendly filter settings, a new file output format in JSON, and lifecycle management of audit logs. @@ -128,11 +128,11 @@ Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, impr For more information, see [documentation](/sql-non-prepared-plan-cache.md). -* Support the DDL distributed parallel execution framework (experimental) [#41495](https://github.com/pingcap/tidb/issues/41495) @[benjamin2037](https://github.com/benjamin2037) +* Support the TiDB Distributed eXecution Framework (DXF) (experimental) [#41495](https://github.com/pingcap/tidb/issues/41495) @[benjamin2037](https://github.com/benjamin2037) - Before TiDB v7.1.0, only one TiDB node can serve as the DDL owner and execute DDL tasks at the same time. Starting from TiDB v7.1.0, in the new distributed parallel execution framework, multiple TiDB nodes can execute the same DDL task in parallel, thus better utilizing the resources of the TiDB cluster and significantly improving the performance of DDL. In addition, you can linearly improve the performance of DDL by adding more TiDB nodes. Note that this feature is currently experimental and only supports `ADD INDEX` operations. + Before TiDB v7.1.0, only one TiDB node can serve as the DDL owner and execute DDL tasks at the same time. Starting from TiDB v7.1.0, in the new DXF, multiple TiDB nodes can execute the same DDL task in parallel, thus better utilizing the resources of the TiDB cluster and significantly improving the performance of DDL. In addition, you can linearly improve the performance of DDL by adding more TiDB nodes. Note that this feature is currently experimental and only supports `ADD INDEX` operations. - To use the distributed framework, set the value of [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) to `ON`: + To use the DXF, set the value of [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) to `ON`: ```sql SET GLOBAL tidb_enable_dist_task = ON; @@ -156,7 +156,7 @@ Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, impr * Support the checkpoint mechanism for Fast Online DDL to improve fault tolerance and automatic recovery capability [#42164](https://github.com/pingcap/tidb/issues/42164) @[tangenta](https://github.com/tangenta) - TiDB v7.1.0 introduces a checkpoint mechanism for [Fast Online DDL](/ddl-introduction.md), which significantly improves the fault tolerance and automatic recovery capability of Fast Online DDL. Even if the TiDB owner node is restarted or changed due to failures, TiDB can still recover progress from checkpoints that are automatically updated on a regular basis, making the DDL execution more stable and efficient. + TiDB v7.1.0 introduces a checkpoint mechanism for [Fast Online DDL](/best-practices/ddl-introduction.md), which significantly improves the fault tolerance and automatic recovery capability of Fast Online DDL. Even if the TiDB owner node is restarted or changed due to failures, TiDB can still recover progress from checkpoints that are automatically updated on a regular basis, making the DDL execution more stable and efficient. For more information, see [documentation](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630). @@ -172,7 +172,7 @@ Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, impr TiDB v7.1.0 introduces lightweight statistics initialization as an experimental feature. Lightweight statistics initialization can significantly reduce the number of statistics that must be loaded during startup, thus improving the speed of loading statistics. This feature increases the stability of TiDB in complex runtime environments and reduces the impact on the overall service when TiDB nodes restart. You can set the parameter [`lite-init-stats`](/tidb-configuration-file.md#lite-init-stats-new-in-v710) to `true` to enable this feature. - During TiDB startup, SQL statements executed before the initial statistics are fully loaded might have suboptimal execution plans, thus causing performance issues. To avoid such issues, TiDB v7.1.0 introduces the configuration parameter [`force-init-stats`](/tidb-configuration-file.md#force-init-stats-new-in-v710). With this option, you can control whether TiDB provides services only after statistics initialization has been finished during startup. This parameter is disabled by default. + During TiDB startup, SQL statements executed before the initial statistics are fully loaded might have suboptimal execution plans, thus causing performance issues. To avoid such issues, TiDB v7.1.0 introduces the configuration parameter [`force-init-stats`](/tidb-configuration-file.md#force-init-stats-new-in-v657-and-v710). With this option, you can control whether TiDB provides services only after statistics initialization has been finished during startup. This parameter is disabled by default. For more information, see [documentation](/statistics.md#load-statistics). @@ -182,7 +182,7 @@ Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, impr For more information, see [documentation](/ticdc/ticdc-integrity-check.md). -* TiCDC optimizes DDL replication operations [#8686](https://github.com/pingcap/tiflow/issues/8686) @[hi-rustin](https://github.com/hi-rustin) +* TiCDC optimizes DDL replication operations [#8686](https://github.com/pingcap/tiflow/issues/8686) @[hi-rustin](https://github.com/Rustin170506) Before v7.1.0, when you perform a DDL operation that affects all rows on a large table (such as adding or deleting a column), the replication latency of TiCDC would significantly increase. Starting from v7.1.0, TiCDC optimizes this replication operation and mitigates the impact of DDL operations on downstream latency. @@ -283,7 +283,7 @@ Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, impr Database auditing is an important feature in TiDB Enterprise Edition. This feature provides a powerful monitoring and auditing tool for enterprises to ensure data security and compliance. It can help enterprise managers in tracking the source and impact of database operations to prevent illegal data theft or tampering. Furthermore, database auditing can also help enterprises meet various regulatory and compliance requirements, ensuring legal and ethical compliance. This feature has important application value for enterprise information security. - This feature is included in TiDB Enterprise Edition. To use this feature and its documentation, navigate to the [TiDB Enterprise](https://www.pingcap.com/tidb-enterprise) page. + For more information, see [user guide](https://static.pingcap.com/files/2023/09/18204824/TiDB-Database-Auditing-User-Guide1.pdf). This feature is included in TiDB Enterprise Edition. To use this feature, navigate to the [TiDB Enterprise](https://www.pingcap.com/tidb-enterprise) page to get TiDB Enterprise Edition. ## Compatibility changes @@ -299,7 +299,7 @@ Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, impr * TiDB Lightning in TiDB versions from v6.2.0 to v7.0.0 decides whether to pause global scheduling based on the TiDB cluster version. When TiDB cluster version >= v6.1.0, scheduling is only paused for the Region that stores the target table data and is resumed after the target table import is complete. While for other versions, TiDB Lightning pauses global scheduling. Starting from TiDB v7.1.0, you can control whether to pause global scheduling by configuring [`pause-pd-scheduler-scope`](/tidb-lightning/tidb-lightning-configuration.md). By default, TiDB Lightning pauses scheduling for the Region that stores the target table data. If the target cluster version is earlier than v6.1.0, an error occurs. In this case, you can change the value of the parameter to `"global"` and try again. -* When you use [`FLASHBACK CLUSTER TO TIMESTAMP`](/sql-statements/sql-statement-flashback-to-timestamp.md) in TiDB v7.1.0, some Regions might remain in the FLASHBACK process even after the completion of the FLASHBACK operation. It is recommended to avoid using this feature in v7.1.0. For more information, see issue [#44292](https://github.com/pingcap/tidb/issues/44292). If you have encountered this issue, you can use the [TiDB snapshot backup and restore](/br/br-snapshot-guide.md) feature to restore data. +* When you use [`FLASHBACK CLUSTER TO TIMESTAMP`](/sql-statements/sql-statement-flashback-cluster.md) in TiDB v7.1.0, some Regions might remain in the FLASHBACK process even after the completion of the FLASHBACK operation. It is recommended to avoid using this feature in v7.1.0. For more information, see issue [#44292](https://github.com/pingcap/tidb/issues/44292). If you have encountered this issue, you can use the [TiDB snapshot backup and restore](/br/br-snapshot-guide.md) feature to restore data. ### System variables @@ -332,10 +332,10 @@ Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, impr | [`authentication_ldap_simple_server_host`](/system-variables.md#authentication_ldap_simple_server_host-new-in-v710) | Newly added | Specifies the LDAP server host in LDAP simple authentication. | | [`authentication_ldap_simple_server_port`](/system-variables.md#authentication_ldap_simple_server_port-new-in-v710) | Newly added | Specifies the LDAP server TCP/IP port number in LDAP simple authentication. | | [`authentication_ldap_simple_tls`](/system-variables.md#authentication_ldap_simple_tls-new-in-v710) | Newly added | Specifies whether connections by the plugin to the LDAP server are protected with StartTLS in LDAP simple authentication. | -| [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) | Newly added | Controls whether to enable the distributed execution framework. After enabling distributed execution, DDL, import, and other supported backend tasks will be jointly completed by multiple TiDB nodes in the cluster. This variable was renamed from `tidb_ddl_distribute_reorg`. | +| [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) | Newly added | Controls whether to enable the Distributed eXecution Framework (DXF). After enabling the DXF, DDL, import, and other supported DXF tasks will be jointly completed by multiple TiDB nodes in the cluster. This variable was renamed from `tidb_ddl_distribute_reorg`. | | [`tidb_enable_non_prepared_plan_cache_for_dml`](/system-variables.md#tidb_enable_non_prepared_plan_cache_for_dml-new-in-v710) | Newly added | Controls whether to enable the [Non-prepared plan cache](/sql-non-prepared-plan-cache.md) feature for DML statements. | | [`tidb_enable_row_level_checksum`](/system-variables.md#tidb_enable_row_level_checksum-new-in-v710) | Newly added | Controls whether to enable the TiCDC data integrity validation for single-row data feature.| -| [`tidb_opt_fix_control`](/system-variables.md#tidb_opt_fix_control-new-in-v710) | Newly added | This variable provides more fine-grained control over the optimizer and helps to prevent performance regression after upgrading caused by behavior changes in the optimizer. | +| [`tidb_opt_fix_control`](/system-variables.md#tidb_opt_fix_control-new-in-v653-and-v710) | Newly added | This variable provides more fine-grained control over the optimizer and helps to prevent performance regression after upgrading caused by behavior changes in the optimizer. | | [`tidb_plan_cache_invalidation_on_fresh_stats`](/system-variables.md#tidb_plan_cache_invalidation_on_fresh_stats-new-in-v710) | Newly added | Controls whether to invalidate the plan cache automatically when statistics on related tables are updated. | | [`tidb_plan_cache_max_plan_size`](/system-variables.md#tidb_plan_cache_max_plan_size-new-in-v710) | Newly added | Controls the maximum size of a plan that can be cached in prepared or non-prepared plan cache. | | [`tidb_prefer_broadcast_join_by_exchange_data_size`](/system-variables.md#tidb_prefer_broadcast_join_by_exchange_data_size-new-in-v710) | Newly added | Controls whether to use the algorithm with the minimum overhead of network transmission. If this variable is enabled, TiDB estimates the size of the data to be exchanged in the network using `Broadcast Hash Join` and `Shuffled Hash Join` respectively, and then chooses the one with the smaller size. [`tidb_broadcast_join_threshold_count`](/system-variables.md#tidb_broadcast_join_threshold_count-new-in-v50) and [`tidb_broadcast_join_threshold_size`](/system-variables.md#tidb_broadcast_join_threshold_size-new-in-v50) will not take effect after this variable is enabled. | @@ -345,7 +345,7 @@ Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, impr | Configuration file | Configuration parameter | Change type | Description | | -------- | -------- | -------- | -------- | -| TiDB | [`performance.force-init-stats`](/tidb-configuration-file.md#force-init-stats-new-in-v710) | Newly added | Controls whether to wait for statistics initialization to finish before providing services during TiDB startup. | +| TiDB | [`performance.force-init-stats`](/tidb-configuration-file.md#force-init-stats-new-in-v657-and-v710) | Newly added | Controls whether to wait for statistics initialization to finish before providing services during TiDB startup. | | TiDB | [`performance.lite-init-stats`](/tidb-configuration-file.md#lite-init-stats-new-in-v710) | Newly added | Controls whether to use lightweight statistics initialization during TiDB startup. | | TiDB | [`log.timeout`](/tidb-configuration-file.md#timeout-new-in-v710) | Newly added | Sets the timeout for log-writing operations in TiDB. In case of a disk failure that prevents logs from being written, this configuration item can trigger the TiDB process to panic instead of hang. The default value is `0`, which means no timeout is set. | | TiKV | [`region-compact-min-redundant-rows`](/tikv-configuration-file.md#region-compact-min-redundant-rows-new-in-v710) | Newly added | Sets the number of redundant MVCC rows required to trigger RocksDB compaction. The default value is `50000`. | @@ -386,7 +386,7 @@ Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, impr + PD - - Add a controller that automatically adjusts the size of the store limit based on the execution details of the snapshot. To enable this controller, set `store-limit-version` to `v2`. Once enabled, you do not need to manually adjust the `store limit` configuration to control the speed of scaling in or scaling out [#6147](https://github.com/tikv/pd/issues/6147) @[bufferflies](https://github.com/bufferflies) + - Add a controller that automatically adjusts the size of the store limit based on the execution details of the snapshot. To enable this controller, set `store-limit-version` to `v2` (experimental). Once enabled, you do not need to manually adjust the `store limit` configuration to control the speed of scaling in or scaling out [#6147](https://github.com/tikv/pd/issues/6147) @[bufferflies](https://github.com/bufferflies) - Add historical load information to avoid frequent scheduling of Regions with unstable loads by the hotspot scheduler when the storage engine is raft-kv2 [#6297](https://github.com/tikv/pd/issues/6297) @[bufferflies](https://github.com/bufferflies) - Add a leader health check mechanism. When the PD server where the etcd leader is located cannot be elected as the leader, PD actively switches the etcd leader to ensure that the PD leader is available [#6403](https://github.com/tikv/pd/issues/6403) @[nolouch](https://github.com/nolouch) @@ -406,10 +406,10 @@ Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, impr - Optimize the directory structure when DDL events occur in the scenario of replicating data to object storage [#8890](https://github.com/pingcap/tiflow/issues/8890) @[CharlesCheung96](https://github.com/CharlesCheung96) - Optimize the method of setting GC TLS for the upstream when the TiCDC replication task fails [#8403](https://github.com/pingcap/tiflow/issues/8403) @[charleszheng44](https://github.com/charleszheng44) - - Support replicating data to the Kafka-on-Pulsar downstream [#8892](https://github.com/pingcap/tiflow/issues/8892) @[hi-rustin](https://github.com/hi-rustin) + - Support replicating data to the Kafka-on-Pulsar downstream [#8892](https://github.com/pingcap/tiflow/issues/8892) @[hi-rustin](https://github.com/Rustin170506) - Support using the open-protocol protocol to only replicate the changed columns after an update occurs when replicating data to Kafka [#8706](https://github.com/pingcap/tiflow/issues/8706) @[sdojjy](https://github.com/sdojjy) - Optimize the error handling of TiCDC in the downstream failures or other scenarios [#8657](https://github.com/pingcap/tiflow/issues/8657) @[hicqu](https://github.com/hicqu) - - Add a configuration item `insecure-skip-verify` to control whether to set the authentication algorithm in the scenario of enabling TLS [#8867](https://github.com/pingcap/tiflow/issues/8867) @[hi-rustin](https://github.com/hi-rustin) + - Add a configuration item `insecure-skip-verify` to control whether to set the authentication algorithm in the scenario of enabling TLS [#8867](https://github.com/pingcap/tiflow/issues/8867) @[hi-rustin](https://github.com/Rustin170506) + TiDB Lightning @@ -425,7 +425,6 @@ Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, impr - Fix the issue of missing table names in the `ADMIN SHOW DDL JOBS` result when a `DROP TABLE` operation is being executed [#42268](https://github.com/pingcap/tidb/issues/42268) @[tiancaiamao](https://github.com/tiancaiamao) - Fix the issue that `Ignore Event Per Minute` and `Stats Cache LRU Cost` charts might not be displayed normally in the Grafana monitoring panel [#42562](https://github.com/pingcap/tidb/issues/42562) @[pingandb](https://github.com/pingandb) - Fix the issue that the `ORDINAL_POSITION` column returns incorrect results when querying the `INFORMATION_SCHEMA.COLUMNS` table [#43379](https://github.com/pingcap/tidb/issues/43379) @[bb7133](https://github.com/bb7133) - - Fix the case sensitivity issue in some columns of the permission table [#41048](https://github.com/pingcap/tidb/issues/41048) @[bb7133](https://github.com/bb7133) - Fix the issue that after a new column is added in the cache table, the value is `NULL` instead of the default value of the column [#42928](https://github.com/pingcap/tidb/issues/42928) @[lqs](https://github.com/lqs) - Fix the issue that CTE results are incorrect when pushing down predicates [#43645](https://github.com/pingcap/tidb/issues/43645) @[winoros](https://github.com/winoros) - Fix the issue of DDL retry caused by write conflict when executing `TRUNCATE TABLE` for partitioned tables with many partitions and TiFlash replicas [#42940](https://github.com/pingcap/tidb/issues/42940) @[mjonss](https://github.com/mjonss) @@ -501,7 +500,7 @@ Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, impr + TiCDC - - Fix the issue of TiCDC time zone setting [#8798](https://github.com/pingcap/tiflow/issues/8798) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue of TiCDC time zone setting [#8798](https://github.com/pingcap/tiflow/issues/8798) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that TiCDC cannot automatically recover when PD address or leader fails [#8812](https://github.com/pingcap/tiflow/issues/8812) [#8877](https://github.com/pingcap/tiflow/issues/8877) @[asddongmen](https://github.com/asddongmen) - Fix the issue that checkpoint lag increases when one of the upstream TiKV nodes crashes [#8858](https://github.com/pingcap/tiflow/issues/8858) @[hicqu](https://github.com/hicqu) - Fix the issue that when replicating data to object storage, the `EXCHANGE PARTITION` operation in the upstream cannot be properly replicated to the downstream [#8914](https://github.com/pingcap/tiflow/issues/8914) @[CharlesCheung96](https://github.com/CharlesCheung96) @@ -534,7 +533,7 @@ Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, impr ## Performance test -To learn about the performance of TiDB v7.1.0, you can refer to the [TPC-C performance test report](https://docs.pingcap.com/tidbcloud/v7.1.0-performance-benchmarking-with-tpcc) and [Sysbench performance test report](https://docs.pingcap.com/tidbcloud/v7.1.0-performance-benchmarking-with-sysbench) of the TiDB Dedicated cluster. +To learn about the performance of TiDB v7.1.0, you can refer to the [TPC-C performance test report](https://docs.pingcap.com/tidbcloud/v7.1.0-performance-benchmarking-with-tpcc) and [Sysbench performance test report](https://docs.pingcap.com/tidbcloud/v7.1.0-performance-benchmarking-with-sysbench) of the TiDB Cloud Dedicated cluster. ## Contributors diff --git a/releases/release-7.1.1.md b/releases/release-7.1.1.md index ad17bb1d0f5bc..cd905ea59ef0c 100644 --- a/releases/release-7.1.1.md +++ b/releases/release-7.1.1.md @@ -9,12 +9,16 @@ Release date: July 24, 2023 TiDB version: 7.1.1 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v7.1.1#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) ## Compatibility changes - TiDB introduces a new system variable `tidb_lock_unchanged_keys` to control whether to lock unchanged keys [#44714](https://github.com/pingcap/tidb/issues/44714) @[ekexium](https://github.com/ekexium) +### Behavior changes + +- When processing update event, TiCDC splits an event into delete and insert events if the primary key or non-null unique index value is modified in the event. For more information, see [documentation](/ticdc/ticdc-split-update-behavior.md#transactions-containing-a-single-update-change). + ## Improvements + TiDB @@ -33,7 +37,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with- + TiCDC - Optimize the encoding format of binary fields when TiCDC replicates data to object storage services [#9373](https://github.com/pingcap/tiflow/issues/9373) @[CharlesCheung96](https://github.com/CharlesCheung96) - - Support the OAUTHBEARER authentication in the scenario of replication to Kafka [#8865](https://github.com/pingcap/tiflow/issues/8865) @[hi-rustin](https://github.com/hi-rustin) + - Support the OAUTHBEARER authentication in the scenario of replication to Kafka [#8865](https://github.com/pingcap/tiflow/issues/8865) @[hi-rustin](https://github.com/Rustin170506) + TiDB Lightning @@ -73,7 +77,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with- - Fix the issue that the cluster upgrade fails when there are paused DDL operations before the upgrade [#44225](https://github.com/pingcap/tidb/issues/44225) @[zimulala](https://github.com/zimulala) - Fix the `duplicate entry` error that occurs when restoring a table with `AUTO_ID_CACHE=1` using BR [#44716](https://github.com/pingcap/tidb/issues/44716) @[tiancaiamao](https://github.com/tiancaiamao) - Fix the data index inconsistency issue triggered by multiple switches of DDL owner [#44619](https://github.com/pingcap/tidb/issues/44619) @[tangenta](https://github.com/tangenta) - - Fix the issue that canceling an `ADD INDEX` DDL task in the `none` status might cause memory leak because this task is not removed from the backend task queue [#44205](https://github.com/pingcap/tidb/issues/44205) @[tangenta](https://github.com/tangenta) + - Fix the issue that canceling an `ADD INDEX` DDL task in the `none` status might cause memory leak because this task is not removed from the Distributed eXecution Framework (DXF) task queue [#44205](https://github.com/pingcap/tidb/issues/44205) @[tangenta](https://github.com/tangenta) - Fix the issue that the proxy protocol reports the `Header read timeout` error when processing certain erroneous data [#43205](https://github.com/pingcap/tidb/issues/43205) @[blacktear23](https://github.com/blacktear23) - Fix the issue that PD isolation might block the running DDL [#44267](https://github.com/pingcap/tidb/issues/44267) @[wjhuang2016](https://github.com/wjhuang2016) - Fix the issue that the query result of the `SELECT CAST(n AS CHAR)` statement is incorrect when `n` in the statement is a negative number [#44786](https://github.com/pingcap/tidb/issues/44786) @[xhebox](https://github.com/xhebox) @@ -111,6 +115,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with- + Backup & Restore (BR) - Fix the issue that `checksum mismatch` is falsely reported in some cases [#44472](https://github.com/pingcap/tidb/issues/44472) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the frequency of `resolve lock` is too high when there is no PITR backup task in the TiDB cluster [#40759](https://github.com/pingcap/tidb/issues/40759) @[joccau](https://github.com/joccau) + TiCDC @@ -118,8 +123,8 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with- - Fix the issue of excessive memory consumption when replicating to an object storage service [#8894](https://github.com/pingcap/tiflow/issues/8894) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the issue that the replication task might get stuck when the redo log is enabled and there is an exception downstream [#9172](https://github.com/pingcap/tiflow/issues/9172) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the issue that TiCDC keeps retrying when there is a downstream failure, which causes the retry time to be too long [#9272](https://github.com/pingcap/tiflow/issues/9272) @[asddongmen](https://github.com/asddongmen) - - Fix the issue of excessive downstream pressure caused by reading downstream metadata too frequently when replicating data to Kafka [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/hi-rustin) - - Fix the issue that when the downstream is Kafka, TiCDC queries the downstream metadata too frequently and causes excessive workload in the downstream [#8957](https://github.com/pingcap/tiflow/issues/8957) [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue of excessive downstream pressure caused by reading downstream metadata too frequently when replicating data to Kafka [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that when the downstream is Kafka, TiCDC queries the downstream metadata too frequently and causes excessive workload in the downstream [#8957](https://github.com/pingcap/tiflow/issues/8957) [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/Rustin170506) - Fix the OOM issue caused by excessive memory usage of the sorter component in some special scenarios [#8974](https://github.com/pingcap/tiflow/issues/8974) @[hicqu](https://github.com/hicqu) - Fix the issue that the `UPDATE` operation cannot output old values when the Avro or CSV protocol is used [#9086](https://github.com/pingcap/tiflow/issues/9086) @[3AceShowHand](https://github.com/3AceShowHand) - Fix the issue that when replicating data to storage services, the JSON file corresponding to downstream DDL statements does not record the default values of table fields [#9066](https://github.com/pingcap/tiflow/issues/9066) @[CharlesCheung96](https://github.com/CharlesCheung96) diff --git a/releases/release-7.1.2.md b/releases/release-7.1.2.md index 99528c3f42241..3ea28aa0d1122 100644 --- a/releases/release-7.1.2.md +++ b/releases/release-7.1.2.md @@ -9,15 +9,21 @@ Release date: October 25, 2023 TiDB version: 7.1.2 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v7.1.2#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) ## Compatibility changes +- Prohibit setting [`require_secure_transport`](https://docs.pingcap.com/tidb/v7.1/system-variables#require_secure_transport-new-in-v610) to `ON` in Security Enhanced Mode (SEM) to prevent potential connectivity issues for users [#47665](https://github.com/pingcap/tidb/issues/47665) @[tiancaiamao](https://github.com/tiancaiamao) +- Disable the [smooth upgrade](/smooth-upgrade-tidb.md) feature by default. You can enable it by sending the `/upgrade/start` and `upgrade/finish` HTTP requests [#47172](https://github.com/pingcap/tidb/issues/47172) @[zimulala](https://github.com/zimulala) - Introduce the [`tidb_opt_enable_hash_join`](https://docs.pingcap.com/tidb/v7.1/system-variables#tidb_opt_enable_hash_join-new-in-v712) system variable to control whether the optimizer selects hash joins for tables [#46695](https://github.com/pingcap/tidb/issues/46695) @[coderplay](https://github.com/coderplay) - Disable periodic compaction of RocksDB by default, so that the default behavior of TiKV RocksDB is now consistent with that in versions before v6.5.0. This change prevents potential performance impact caused by a significant number of compactions after upgrading. In addition, TiKV introduces two new configuration items [`rocksdb.[defaultcf|writecf|lockcf].periodic-compaction-seconds`](https://docs.pingcap.com/tidb/v7.1/tikv-configuration-file#periodic-compaction-seconds-new-in-v712) and [`rocksdb.[defaultcf|writecf|lockcf].ttl`](https://docs.pingcap.com/tidb/v7.1/tikv-configuration-file#ttl-new-in-v712), enabling you to manually configure periodic compaction of RocksDB [#15355](https://github.com/tikv/tikv/issues/15355) @[LykxSassinator](https://github.com/LykxSassinator) - TiCDC introduces the [`sink.csv.binary-encoding-method`](/ticdc/ticdc-changefeed-config.md#changefeed-configuration-parameters) configuration item to control the encoding method of binary data in the CSV protocol. The default value is `'base64'` [#9373](https://github.com/pingcap/tiflow/issues/9373) @[CharlesCheung96](https://github.com/CharlesCheung96) - TiCDC introduces the [`large-message-handle-option`](/ticdc/ticdc-sink-to-kafka.md#handle-messages-that-exceed-the-kafka-topic-limit) configuration item. It is empty by default, which means that the changefeed fails when the message size exceeds the limit of the Kafka topic. When this configuration is set to `"handle-key-only"`, if the message exceeds the size limit, only the handle key will be sent to reduce the message size; if the reduced message still exceeds the limit, then the changefeed fails [#9680](https://github.com/pingcap/tiflow/issues/9680) @[3AceShowHand](https://github.com/3AceShowHand) +### Behavior changes + +- For transactions containing multiple changes, if the primary key or non-null unique index value is modified in the update event, TiCDC splits an event into delete and insert events and ensures that all events follow the sequence of delete events preceding insert events. For more information, see [documentation](/ticdc/ticdc-split-update-behavior.md#transactions-containing-multiple-update-changes). + ## Improvements + TiDB @@ -102,12 +108,12 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with- - Fix the issue that inserting data into a partitioned table might fail after exchanging partitions between the partition table and a table with placement policies [#45791](https://github.com/pingcap/tidb/issues/45791) @[mjonss](https://github.com/mjonss) - Fix the issue of encoding time fields with incorrect timezone information [#46033](https://github.com/pingcap/tidb/issues/46033) @[tangenta](https://github.com/tangenta) - Fix the issue that DDL statements that fast add indexes would get stuck when the `tmp` directory does not exist [#45456](https://github.com/pingcap/tidb/issues/45456) @[tangenta](https://github.com/tangenta) - - Fix the issue that upgrading multiple TiDB instances simultaneously might block the upgrade process [#46288](https://github.com/pingcap/tidb/issues/46228) @[zimulala](https://github.com/zimulala) + - Fix the issue that upgrading multiple TiDB instances simultaneously might block the upgrade process [#46228](https://github.com/pingcap/tidb/issues/46228) @[zimulala](https://github.com/zimulala) - Fix the issue of uneven Region scattering caused by incorrect parameters used in splitting Regions [#46135](https://github.com/pingcap/tidb/issues/46135) @[zimulala](https://github.com/zimulala) - Fix the issue that DDL operations might get stuck after TiDB is restarted [#46751](https://github.com/pingcap/tidb/issues/46751) @[wjhuang2016](https://github.com/wjhuang2016) - Prohibit split table operations on non-integer clustered indexes [#47350](https://github.com/pingcap/tidb/issues/47350) @[tangenta](https://github.com/tangenta) - Fix the issue that DDL operations might get permanently blocked due to incorrect MDL handling [#46920](https://github.com/pingcap/tidb/issues/46920) @[wjhuang2016](https://github.com/wjhuang2016) - - Fix the issue of duplicate columns in a table caused by `RENAME TABLE` operations [#47064](https://github.com/pingcap/tidb/issues/47064)@[jiyfhust](https://github.com/jiyfhust) + - Fix the issue of duplicate rows in `information_schema.columns` caused by renaming a table [#47064](https://github.com/pingcap/tidb/issues/47064) @[jiyfhust](https://github.com/jiyfhust) - Fix the panic issue of `batch-client` in `client-go` [#47691](https://github.com/pingcap/tidb/issues/47691) @[crazycs520](https://github.com/crazycs520) - Fix the issue that statistics collection on partitioned tables is not killed in time when its memory usage exceeds memory limits [#45706](https://github.com/pingcap/tidb/issues/45706) @[hawkingrei](https://github.com/hawkingrei) - Fix the issue that query results are inaccurate when queries contain `UNHEX` conditions [#45378](https://github.com/pingcap/tidb/issues/45378) @[qw4990](https://github.com/qw4990) @@ -163,7 +169,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with- - Fix the issue that PITR fails to recover data from GCS [#47022](https://github.com/pingcap/tidb/issues/47022) @[Leavrth](https://github.com/Leavrth) - Fix the potential error in fine-grained backup phase in RawKV mode [#37085](https://github.com/pingcap/tidb/issues/37085) @[pingyu](https://github.com/pingyu) - Fix the issue that recovering meta-kv using PITR might cause errors [#46578](https://github.com/pingcap/tidb/issues/46578) @[Leavrth](https://github.com/Leavrth) - - Fix the errors in BR integration test cases [#45561](https://github.com/pingcap/tidb/issues/46561) @[purelind](https://github.com/purelind) + - Fix the errors in BR integration test cases [#46561](https://github.com/pingcap/tidb/issues/46561) @[purelind](https://github.com/purelind) - Fix the issue of restore failures by increasing the default values of the global parameters `TableColumnCountLimit` and `IndexLimit` used by BR to their maximum values [#45793](https://github.com/pingcap/tidb/issues/45793) @[Leavrth](https://github.com/Leavrth) - Fix the issue that the br CLI client gets stuck when scanning restored data [#45476](https://github.com/pingcap/tidb/issues/45476) @[3pointer](https://github.com/3pointer) - Fix the issue that PITR might skip restoring the `CREATE INDEX` DDL statement [#47482](https://github.com/pingcap/tidb/issues/47482) @[Leavrth](https://github.com/Leavrth) @@ -189,6 +195,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with- - Fix the issue that replication write conflicts might occur when the unique keys for multiple rows are modified in one transaction on the upstream [#9430](https://github.com/pingcap/tiflow/issues/9430) @[sdojjy](https://github.com/sdojjy) - Fix the issue that the replication task might get stuck when the downstream encounters a short-term failure [#9542](https://github.com/pingcap/tiflow/issues/9542) [#9272](https://github.com/pingcap/tiflow/issues/9272) [#9582](https://github.com/pingcap/tiflow/issues/9582) [#9592](https://github.com/pingcap/tiflow/issues/9592) @[hicqu](https://github.com/hicqu) - Fix the issue that the replication task might get stuck when the downstream encounters an error and retries [#9450](https://github.com/pingcap/tiflow/issues/9450) @[hicqu](https://github.com/hicqu) + - Fix the issue that TiCDC might get stuck when replicating data to Kafka [#9855](https://github.com/pingcap/tiflow/issues/9855) @[hicqu](https://github.com/hicqu) + TiDB Data Migration (DM) diff --git a/releases/release-7.1.3.md b/releases/release-7.1.3.md new file mode 100644 index 0000000000000..11848f7b57df5 --- /dev/null +++ b/releases/release-7.1.3.md @@ -0,0 +1,162 @@ +--- +title: TiDB 7.1.3 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 7.1.3. +--- + +# TiDB 7.1.3 Release Notes + +Release date: December 21, 2023 + +TiDB version: 7.1.3 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) + +## Compatibility changes + +- After further testing, the default value of the TiCDC Changefeed configuration item [`case-sensitive`](/ticdc/ticdc-changefeed-config.md) is changed from `true` to `false`. This means that by default, table and database names in the TiCDC configuration file are case-insensitive [#10047](https://github.com/pingcap/tiflow/issues/10047) @[sdojjy](https://github.com/sdojjy) +- TiCDC Changefeed introduces the following new configuration items: + - [`sql-mode`](/ticdc/ticdc-changefeed-config.md): enables you to set the [SQL mode](/ticdc/ticdc-ddl.md#sql-mode) used by TiCDC to parse DDL statements when TiCDC replicates data [#9876](https://github.com/pingcap/tiflow/issues/9876) @[asddongmen](https://github.com/asddongmen) + - [`encoding-worker-num`](/ticdc/ticdc-changefeed-config.md) and [`flush-worker-num`](/ticdc/ticdc-changefeed-config.md): enables you to set different concurrency parameters for the redo module based on specifications of different machines [#10048](https://github.com/pingcap/tiflow/issues/10048) @[CharlesCheung96](https://github.com/CharlesCheung96) + - [`compression`](/ticdc/ticdc-changefeed-config.md): enables you to configure the compression behavior of redo log files [#10176](https://github.com/pingcap/tiflow/issues/10176) @[sdojjy](https://github.com/sdojjy) + - [`sink.cloud-storage-config`](/ticdc/ticdc-changefeed-config.md): enables you to set the automatic cleanup of historical data when replicating data to object storage [#10109](https://github.com/pingcap/tiflow/issues/10109) @[CharlesCheung96](https://github.com/CharlesCheung96) + +## Improvements + ++ TiDB + + - Support the [`FLASHBACK CLUSTER TO TSO`](https://docs.pingcap.com/tidb/v7.1/sql-statement-flashback-cluster) syntax [#48372](https://github.com/pingcap/tidb/issues/48372) @[BornChanger](https://github.com/BornChanger) + ++ PD + + - Enhance the configuration retrieval method of the resource control client to dynamically fetch the latest configurations [#7043](https://github.com/tikv/pd/issues/7043) @[nolouch](https://github.com/nolouch) + ++ Tools + + + Backup & Restore (BR) + + - Enable automatic retry of Region scatter during snapshot recovery when encountering timeout failures or cancellations of Region scatter [#47236](https://github.com/pingcap/tidb/issues/47236) @[Leavrth](https://github.com/Leavrth) + - During restoring a snapshot backup, BR retries when it encounters certain network errors [#48528](https://github.com/pingcap/tidb/issues/48528) @[Leavrth](https://github.com/Leavrth) + - Introduce a new integration test for Point-In-Time Recovery (PITR) in the `delete range` scenario, enhancing PITR stability [#47738](https://github.com/pingcap/tidb/issues/47738) @[Leavrth](https://github.com/Leavrth) + + + TiCDC + + - Optimize the memory consumption when TiCDC nodes replicate data to TiDB [#9935](https://github.com/pingcap/tiflow/issues/9935) @[3AceShowHand](https://github.com/3AceShowHand) + - Optimize some alarm rules [#9266](https://github.com/pingcap/tiflow/issues/9266) @[asddongmen](https://github.com/asddongmen) + - Optimize the performance of redo log, including parallel writing data to S3 and adopting lz4 compression algorithm [#10176](https://github.com/pingcap/tiflow/issues/10176) [#10226](https://github.com/pingcap/tiflow/issues/10226) @[sdojjy](https://github.com/sdojjy) + - Improve the performance of TiCDC replicating data to object storage by increasing parallelism [#10098](https://github.com/pingcap/tiflow/issues/10098) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Reduce the impact of TiCDC incremental scanning on upstream TiKV [#11390](https://github.com/tikv/tikv/issues/11390) @[hicqu](https://github.com/hicqu) + - Support making TiCDC Canal-JSON content format [compatible with the content format of the official Canal output](https://docs.pingcap.com/tidb/v6.5/ticdc-canal-json#compatibility-with-the-official-canal) by setting `content-compatible=true` in the `sink-uri` configuration [#10106](https://github.com/pingcap/tiflow/issues/10106) @[3AceShowHand](https://github.com/3AceShowHand) + + + TiDB Lightning + + - Add a retry mechanism for `GetTS` failure caused by PD leader change [#45301](https://github.com/pingcap/tidb/issues/45301) @[lance6716](https://github.com/lance6716) + +## Bug fixes + ++ TiDB + + - Fix the issue that queries containing common table expressions (CTEs) unexpectedly get stuck when the memory limit is exceeded [#49096](https://github.com/pingcap/tidb/issues/49096) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that high CPU usage of TiDB occurs due to long-term memory pressure caused by `tidb_server_memory_limit` [#48741](https://github.com/pingcap/tidb/issues/48741) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that queries containing CTEs report `runtime error: index out of range [32] with length 32` when `tidb_max_chunk_size` is set to a small value [#48808](https://github.com/pingcap/tidb/issues/48808) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that the query result is incorrect when an `ENUM` type column is used as the join key [#48991](https://github.com/pingcap/tidb/issues/48991) @[winoros](https://github.com/winoros) + - Fix the parsing error caused by aggregate or window functions in recursive CTEs [#47711](https://github.com/pingcap/tidb/issues/47711) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that `UPDATE` statements might be incorrectly converted to PointGet [#47445](https://github.com/pingcap/tidb/issues/47445) @[hi-rustin](https://github.com/Rustin170506) + - Fix the OOM issue that might occur when TiDB performs garbage collection on the `stats_history` table [#48431](https://github.com/pingcap/tidb/issues/48431) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the same query plan has different `PLAN_DIGEST` values in some cases [#47634](https://github.com/pingcap/tidb/issues/47634) @[King-Dylan](https://github.com/King-Dylan) + - Fix the issue that `GenJSONTableFromStats` cannot be killed when it consumes a large amount of memory [#47779](https://github.com/pingcap/tidb/issues/47779) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the result might be incorrect when predicates are pushed down to common table expressions [#47881](https://github.com/pingcap/tidb/issues/47881) @[winoros](https://github.com/winoros) + - Fix the issue that `Duplicate entry` might occur when `AUTO_ID_CACHE=1` is set [#46444](https://github.com/pingcap/tidb/issues/46444) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that TiDB server might consume a significant amount of resources when the enterprise plugin for audit logging is used [#49273](https://github.com/pingcap/tidb/issues/49273) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that TiDB server might panic during graceful shutdown [#36793](https://github.com/pingcap/tidb/issues/36793) @[bb7133](https://github.com/bb7133) + - Fix the issue that tables with `AUTO_ID_CACHE=1` might lead to gRPC client leaks when there are a large number of tables [#48869](https://github.com/pingcap/tidb/issues/48869) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the incorrect error message for `ErrLoadDataInvalidURI` (invalid S3 URI error) [#48164](https://github.com/pingcap/tidb/issues/48164) @[lance6716](https://github.com/lance6716) + - Fix the issue that executing `ALTER TABLE ... LAST PARTITION` fails when the partition column type is `DATETIME` [#48814](https://github.com/pingcap/tidb/issues/48814) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that the actual error message during `IMPORT INTO` execution might be overridden by other error messages [#47992](https://github.com/pingcap/tidb/issues/47992) [#47781](https://github.com/pingcap/tidb/issues/47781) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that TiDB deployed in the cgroup v2 container cannot be detected [#48342](https://github.com/pingcap/tidb/issues/48342) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that executing `UNION ALL` with the DUAL table as the first subnode might cause an error [#48755](https://github.com/pingcap/tidb/issues/48755) @[winoros](https://github.com/winoros) + - Fix the TiDB node panic issue that occurs when DDL `jobID` is restored to 0 [#46296](https://github.com/pingcap/tidb/issues/46296) @[jiyfhust](https://github.com/jiyfhust) + - Fix the issue of unsorted row data returned by `TABLESAMPLE` [#48253](https://github.com/pingcap/tidb/issues/48253) @[tangenta](https://github.com/tangenta) + - Fix the issue that panic might occur when `tidb_enable_ordered_result_mode` is enabled [#45044](https://github.com/pingcap/tidb/issues/45044) @[qw4990](https://github.com/qw4990) + - Fix the issue that the optimizer mistakenly selects IndexFullScan to reduce sort introduced by window functions [#46177](https://github.com/pingcap/tidb/issues/46177) @[qw4990](https://github.com/qw4990) + - Fix the issue of not handling locks in the MVCC interface when reading schema diff commit versions from the TiDB schema cache [#48281](https://github.com/pingcap/tidb/issues/48281) @[cfzjywxk](https://github.com/cfzjywxk) + - Fix the issue of incorrect memory usage estimation in `INDEX_LOOKUP_HASH_JOIN` [#47788](https://github.com/pingcap/tidb/issues/47788) @[SeaRise](https://github.com/SeaRise) + - Fix the issue of `IMPORT INTO` task failure caused by PD leader malfunction for 1 minute [#48307](https://github.com/pingcap/tidb/issues/48307) @[D3Hunter](https://github.com/D3Hunter) + - Fix the panic issue of `batch-client` in `client-go` [#47691](https://github.com/pingcap/tidb/issues/47691) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that column pruning can cause panic in specific situations [#47331](https://github.com/pingcap/tidb/issues/47331) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that TiDB does not read `cgroup` resource limits when it is started with `systemd` [#47442](https://github.com/pingcap/tidb/issues/47442) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue of possible syntax error when a common table expression (CTE) containing aggregate or window functions is referenced by other recursive CTEs [#47603](https://github.com/pingcap/tidb/issues/47603) [#47711](https://github.com/pingcap/tidb/issues/47711) @[elsa0520](https://github.com/elsa0520) + - Fix the panic issue that might occur when constructing TopN structure for statistics [#35948](https://github.com/pingcap/tidb/issues/35948) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that the result of `COUNT(INT)` calculated by MPP might be incorrect [#48643](https://github.com/pingcap/tidb/issues/48643) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that the chunk cannot be reused when the HashJoin operator performs probe [#48082](https://github.com/pingcap/tidb/issues/48082) @[wshwsh12](https://github.com/wshwsh12) + ++ TiKV + + - Fix the issue that if TiKV runs extremely slowly, it might panic after Region merge [#16111](https://github.com/tikv/tikv/issues/16111) @[overvenus](https://github.com/overvenus) + - Fix the issue that Resolved TS might be blocked for two hours [#15520](https://github.com/tikv/tikv/issues/15520) [#39130](https://github.com/pingcap/tidb/issues/39130) @[overvenus](https://github.com/overvenus) + - Fix the issue that TiKV reports the `ServerIsBusy` error because it can not append the raft log [#15800](https://github.com/tikv/tikv/issues/15800) @[tonyxuqqi](https://github.com/tonyxuqqi) + - Fix the issue that snapshot restore might get stuck when BR crashes [#15684](https://github.com/tikv/tikv/issues/15684) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that Resolved TS in stale read might cause TiKV OOM issues when tracking large transactions [#14864](https://github.com/tikv/tikv/issues/14864) @[overvenus](https://github.com/overvenus) + - Fix the issue that damaged SST files might be spread to other TiKV nodes [#15986](https://github.com/tikv/tikv/issues/15986) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that the joint state of DR Auto-Sync might time out when scaling out [#15817](https://github.com/tikv/tikv/issues/15817) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that the scheduler command variables are incorrect in Grafana on the cloud environment [#15832](https://github.com/tikv/tikv/issues/15832) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that stale peers are retained and block resolved-ts after Regions are merged [#15919](https://github.com/tikv/tikv/issues/15919) @[overvenus](https://github.com/overvenus) + - Fix the issue that Online Unsafe Recovery cannot handle merge abort [#15580](https://github.com/tikv/tikv/issues/15580) @[v01dstar](https://github.com/v01dstar) + - Fix the TiKV OOM issue that occurs when restarting TiKV and there are a large number of Raft logs that are not applied [#15770](https://github.com/tikv/tikv/issues/15770) @[overvenus](https://github.com/overvenus) + - Fix security issues by upgrading the version of `lz4-sys` to 1.9.4 [#15621](https://github.com/tikv/tikv/issues/15621) @[SpadeA-Tang](https://github.com/SpadeA-Tang) + - Fix the issue that `blob-run-mode` in Titan cannot be updated online [#15978](https://github.com/tikv/tikv/issues/15978) @[tonyxuqqi](https://github.com/tonyxuqqi) + - Fix the issue that network interruption between PD and TiKV might cause PITR to get stuck [#15279](https://github.com/tikv/tikv/issues/15279) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that TiKV coprocessor might return stale data when removing a Raft peer [#16069](https://github.com/tikv/tikv/issues/16069) @[overvenus](https://github.com/overvenus) + ++ PD + + - Fix the issue that the `resource_manager_resource_unit` metric is empty in TiDB Dashboard when executing `CALIBRATE RESOURCE` [#45166](https://github.com/pingcap/tidb/issues/45166) @[CabinfeverB](https://github.com/CabinfeverB) + - Fix the issue that the Calibrate by Workload page reports an error [#48162](https://github.com/pingcap/tidb/issues/48162) @[CabinfeverB](https://github.com/CabinfeverB) + - Fix the issue that deleting a resource group can damage DDL atomicity [#45050](https://github.com/pingcap/tidb/issues/45050) @[glorv](https://github.com/glorv) + - Fix the issue that when PD leader is transferred and there is a network partition between the new leader and the PD client, the PD client fails to update the information of the leader [#7416](https://github.com/tikv/pd/issues/7416) @[CabinfeverB](https://github.com/CabinfeverB) + - Fix the issue that adding multiple TiKV nodes to a large cluster might cause TiKV heartbeat reporting to become slow or stuck [#7248](https://github.com/tikv/pd/issues/7248) @[rleungx](https://github.com/rleungx) + - Fix the issue that TiDB Dashboard cannot read PD `trace` data correctly [#7253](https://github.com/tikv/pd/issues/7253) @[nolouch](https://github.com/nolouch) + - Fix some security issues by upgrading the version of Gin Web Framework from v1.8.1 to v1.9.1 [#7438](https://github.com/tikv/pd/issues/7438) @[niubell](https://github.com/niubell) + - Fix the issue that the rule checker does not add Learners according to the configuration of Placement Rules [#7185](https://github.com/tikv/pd/issues/7185) @[nolouch](https://github.com/nolouch) + - Fix the issue that PD might delete normal Peers when TiKV nodes are unavailable [#7249](https://github.com/tikv/pd/issues/7249) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that it takes a long time to switch the leader in DR Auto-Sync mode [#6988](https://github.com/tikv/pd/issues/6988) @[HuSharp](https://github.com/HuSharp) + ++ TiFlash + + - Fix the issue that executing the `ALTER TABLE ... EXCHANGE PARTITION ...` statement causes panic [#8372](https://github.com/pingcap/tiflash/issues/8372) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue of memory leak when TiFlash encounters memory limitation during query [#8447](https://github.com/pingcap/tiflash/issues/8447) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that data of TiFlash replicas would still be garbage collected after executing `FLASHBACK DATABASE` [#8450](https://github.com/pingcap/tiflash/issues/8450) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix incorrect display of maximum percentile time for some panels in Grafana [#8076](https://github.com/pingcap/tiflash/issues/8076) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that a query returns the unexpected error message "Block schema mismatch in FineGrainedShuffleWriter-V1" [#8111](https://github.com/pingcap/tiflash/issues/8111) @[SeaRise](https://github.com/SeaRise) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that the default values for BR SQL commands and CLI are different, which might cause OOM issues [#48000](https://github.com/pingcap/tidb/issues/48000) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that the log backup might get stuck in some scenarios when backing up large wide tables [#15714](https://github.com/tikv/tikv/issues/15714) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that BR generates incorrect URIs for external storage files [#48452](https://github.com/pingcap/tidb/issues/48452) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that the retry after an EC2 metadata connection reset causes degraded backup and restore performance [#47650](https://github.com/pingcap/tidb/issues/47650) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the log backup task can start but does not work properly if failing to connect to PD during task initialization [#16056](https://github.com/tikv/tikv/issues/16056) @[YuJuncen](https://github.com/YuJuncen) + + + TiCDC + + - Fix the issue that the `WHERE` clause does not use the primary key as a condition when replicating `DELETE` statements in certain scenarios [#9812](https://github.com/pingcap/tiflow/issues/9812) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that replication tasks get stuck in certain special scenarios when replicating data to object storage [#10041](https://github.com/pingcap/tiflow/issues/10041) [#10044](https://github.com/pingcap/tiflow/issues/10044) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that replication tasks get stuck in certain special scenarios after enabling sync-point and redo log [#10091](https://github.com/pingcap/tiflow/issues/10091) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that TiCDC mistakenly closes the connection with TiKV in certain special scenarios [#10239](https://github.com/pingcap/tiflow/issues/10239) @[hicqu](https://github.com/hicqu) + - Fix the issue that the changefeed cannot replicate DML events in bidirectional replication mode if the target table is dropped and then recreated in upstream [#10079](https://github.com/pingcap/tiflow/issues/10079) @[asddongmen](https://github.com/asddongmen) + - Fix the performance issue caused by accessing NFS directories when replicating data to an object store sink [#10041](https://github.com/pingcap/tiflow/issues/10041) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that the TiCDC server might panic when replicating data to an object storage service [#10137](https://github.com/pingcap/tiflow/issues/10137) @[sdojjy](https://github.com/sdojjy) + - Fix the issue that the interval between replicating DDL statements is too long when redo log is enabled [#9960](https://github.com/pingcap/tiflow/issues/9960) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that an owner node gets stuck due to NFS failure when the redo log is enabled [#9886](https://github.com/pingcap/tiflow/issues/9886) @[3AceShowHand](https://github.com/3AceShowHand) + + + TiDB Data Migration (DM) + + - Fix the issue that DM performance is degraded due to an inappropriate algorithm used for `CompareGTID` [#9676](https://github.com/pingcap/tiflow/issues/9676) @[GMHDBJD](https://github.com/GMHDBJD) + + + TiDB Lightning + + - Fix the issue that data import fails due to the PD leader being killed or slow processing of PD requests [#46950](https://github.com/pingcap/tidb/issues/46950) [#48075](https://github.com/pingcap/tidb/issues/48075) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that TiDB Lightning gets stuck during `writeToTiKV` [#46321](https://github.com/pingcap/tidb/issues/46321) [#48352](https://github.com/pingcap/tidb/issues/48352) @[lance6716](https://github.com/lance6716) + - Fix the issue that data import fails because HTTP retry requests do not use the current request content [#47930](https://github.com/pingcap/tidb/issues/47930) @[lance6716](https://github.com/lance6716) + - Remove unnecessary `get_regions` calls in physical import mode [#45507](https://github.com/pingcap/tidb/issues/45507) @[mittalrishabh](https://github.com/mittalrishabh) diff --git a/releases/release-7.1.4.md b/releases/release-7.1.4.md new file mode 100644 index 0000000000000..9f7629016e269 --- /dev/null +++ b/releases/release-7.1.4.md @@ -0,0 +1,185 @@ +--- +title: TiDB 7.1.4 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 7.1.4. +--- + +# TiDB 7.1.4 Release Notes + +Release date: March 11, 2024 + +TiDB version: 7.1.4 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) + +## Compatibility changes + +- To reduce the overhead of log printing, TiFlash changes the default value of `logger.level` from `"debug"` to `"info"` [#8641](https://github.com/pingcap/tiflash/issues/8641) @[JaySon-Huang](https://github.com/JaySon-Huang) +- Introduce the TiKV configuration item [`gc.num-threads`](https://docs.pingcap.com/tidb/v6.5/tikv-configuration-file#num-threads-new-in-v658) to set the number of GC threads when `enable-compaction-filter` is `false` [#16101](https://github.com/tikv/tikv/issues/16101) @[tonyxuqqi](https://github.com/tonyxuqqi) + +## Improvements + ++ TiDB + + - Enhance the ability to convert `OUTER JOIN` to `INNER JOIN` in specific scenarios [#49616](https://github.com/pingcap/tidb/issues/49616) @[qw4990](https://github.com/qw4990) + - When `force-init-stats` is set to `true`, TiDB waits for statistics initialization to finish before providing services during TiDB startup. This setting no longer blocks the startup of HTTP servers, which enables users to continue monitoring [#50854](https://github.com/pingcap/tidb/issues/50854) @[hawkingrei](https://github.com/hawkingrei) + ++ TiKV + + - When TiKV detects the existence of corrupted SST files, it logs the specific reasons for the corruption [#16308](https://github.com/tikv/tikv/issues/16308) @[overvenus](https://github.com/overvenus) + ++ PD + + - Improve the speed of PD automatically updating cluster status when the backup cluster is disconnected [#6883](https://github.com/tikv/pd/issues/6883) @[disksing](https://github.com/disksing) + ++ TiFlash + + - Reduce the impact of background GC tasks on read and write task latency [#8650](https://github.com/pingcap/tiflash/issues/8650) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Reduce the impact of disk performance jitter on read latency [#8583](https://github.com/pingcap/tiflash/issues/8583) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - Support creating databases in batch during data restore [#50767](https://github.com/pingcap/tidb/issues/50767) @[Leavrth](https://github.com/Leavrth) + - Improve the table creation performance of the `RESTORE` statement in scenarios with large datasets [#48301](https://github.com/pingcap/tidb/issues/48301) @[Leavrth](https://github.com/Leavrth) + - Improve the speed of merging SST files during data restore by using a more efficient algorithm [#50613](https://github.com/pingcap/tidb/issues/50613) @[Leavrth](https://github.com/Leavrth) + - Support ingesting SST files in batch during data restore [#16267](https://github.com/tikv/tikv/issues/16267) @[3pointer](https://github.com/3pointer) + - Print the information of the slowest Region that affects global checkpoint advancement in logs and metrics during log backups [#51046](https://github.com/pingcap/tidb/issues/51046) @[YuJuncen](https://github.com/YuJuncen) + - Remove an outdated compatibility check when using Google Cloud Storage (GCS) as the external storage [#50533](https://github.com/pingcap/tidb/issues/50533) @[lance6716](https://github.com/lance6716) + - Implement a lock mechanism to avoid executing multiple log backup truncation tasks (`br log truncate`) simultaneously [#49414](https://github.com/pingcap/tidb/issues/49414) @[YuJuncen](https://github.com/YuJuncen) + + + TiCDC + + - When the downstream is Kafka, the topic expression allows `schema` to be optional and supports specifying a topic name directly [#9763](https://github.com/pingcap/tiflow/issues/9763) @[3AceShowHand](https://github.com/3AceShowHand) + - Support [querying the downstream synchronization status of a changefeed](https://docs.pingcap.com/tidb/v7.1/ticdc-open-api-v2#query-whether-a-specific-replication-task-is-completed), which helps you determine whether the upstream data changes received by TiCDC have been synchronized to the downstream system completely [#10289](https://github.com/pingcap/tiflow/issues/10289) @[hongyunyan](https://github.com/hongyunyan) + - Support searching TiCDC logs in the TiDB Dashboard [#10263](https://github.com/pingcap/tiflow/issues/10263) @[CharlesCheung96](https://github.com/CharlesCheung96) + + + TiDB Lightning + + - Improve the performance in scenarios where multiple tables are imported by removing the lock operation when executing `ALTER TABLE` [#50105](https://github.com/pingcap/tidb/issues/50105) @[D3Hunter](https://github.com/D3Hunter) + +## Bug fixes + ++ TiDB + + - Fix the issue that the `DELETE` and `UPDATE` statements using index lookup might report an error when `tidb_multi_statement_mode` mode is enabled [#50012](https://github.com/pingcap/tidb/issues/50012) @[tangenta](https://github.com/tangenta) + - Fix the issue that CTE queries might report an error `type assertion for CTEStorageMap failed` during the retry process [#46522](https://github.com/pingcap/tidb/issues/46522) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue of excessive statistical error in constructing statistics caused by Golang's implicit conversion algorithm [#49801](https://github.com/pingcap/tidb/issues/49801) @[qw4990](https://github.com/qw4990) + - Fix the issue that errors might be returned during the concurrent merging of global statistics for partitioned tables [#48713](https://github.com/pingcap/tidb/issues/48713) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue of wrong query results due to TiDB incorrectly eliminating constant values in `group by` [#38756](https://github.com/pingcap/tidb/issues/38756) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that `BIT` type columns might cause query errors due to decode failures when they are involved in calculations of some functions [#49566](https://github.com/pingcap/tidb/issues/49566) [#50850](https://github.com/pingcap/tidb/issues/50850) [#50855](https://github.com/pingcap/tidb/issues/50855) @[jiyfhust](https://github.com/jiyfhust) + - Fix the issue that `LIMIT` in multi-level nested `UNION` queries might become ineffective [#49874](https://github.com/pingcap/tidb/issues/49874) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that the auto-increment ID allocation reports an error due to concurrent conflicts when using an auto-increment column with `AUTO_ID_CACHE=1` [#50519](https://github.com/pingcap/tidb/issues/50519) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the `Column ... in from clause is ambiguous` error that might occur when a query uses `NATURAL JOIN` [#32044](https://github.com/pingcap/tidb/issues/32044) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that enforced sorting might become ineffective when a query uses optimizer hints (such as `STREAM_AGG()`) that enforce sorting and its execution plan contains `IndexMerge` [#49605](https://github.com/pingcap/tidb/issues/49605) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that query results are incorrect due to `STREAM_AGG()` incorrectly handling CI [#49902](https://github.com/pingcap/tidb/issues/49902) @[wshwsh12](https://github.com/wshwsh12) + - Fix the goroutine leak issue that might occur when the `HashJoin` operator fails to spill to disk [#50841](https://github.com/pingcap/tidb/issues/50841) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that hints cannot be used in `REPLACE INTO` statements [#34325](https://github.com/pingcap/tidb/issues/34325) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that executing queries containing the `GROUP_CONCAT(ORDER BY)` syntax might return errors [#49986](https://github.com/pingcap/tidb/issues/49986) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that using a multi-valued index to access an empty JSON array might return incorrect results [#50125](https://github.com/pingcap/tidb/issues/50125) @[YangKeao](https://github.com/YangKeao) + - Fix the goroutine leak issue that occurs when the memory usage of CTE queries exceeds limits [#50337](https://github.com/pingcap/tidb/issues/50337) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that using old interfaces might cause inconsistent metadata for tables [#49751](https://github.com/pingcap/tidb/issues/49751) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that executing `UNIQUE` index lookup with an `ORDER BY` clause might cause an error [#49920](https://github.com/pingcap/tidb/issues/49920) @[jackysp](https://github.com/jackysp) + - Fix the issue that common hints do not take effect in `UNION ALL` statements [#50068](https://github.com/pingcap/tidb/issues/50068) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that a query containing the IndexHashJoin operator gets stuck when memory exceeds `tidb_mem_quota_query` [#49033](https://github.com/pingcap/tidb/issues/49033) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that `UPDATE` or `DELETE` statements containing `WITH RECURSIVE` CTEs might produce incorrect results [#48969](https://github.com/pingcap/tidb/issues/48969) @[winoros](https://github.com/winoros) + - Fix the issue that histogram statistics might not be parsed into readable strings when the histogram boundary contains `NULL` [#49823](https://github.com/pingcap/tidb/issues/49823) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that TiDB might panic when a query contains the Apply operator and the `fatal error: concurrent map writes` error occurs [#50347](https://github.com/pingcap/tidb/issues/50347) @[SeaRise](https://github.com/SeaRise) + - Fix the `Can't find column ...` error that might occur when aggregate functions are used for group calculations [#50926](https://github.com/pingcap/tidb/issues/50926) @[qw4990](https://github.com/qw4990) + - Fix the issue that TiDB returns wrong query results when processing `ENUM` or `SET` types by constant propagation [#49440](https://github.com/pingcap/tidb/issues/49440) @[winoros](https://github.com/winoros) + - Fix the issue that the completion times of two DDL tasks with dependencies are incorrectly sequenced [#49498](https://github.com/pingcap/tidb/issues/49498) @[tangenta](https://github.com/tangenta) + - Fix the issue that TiDB might panic when using the `EXECUTE` statement to execute `PREPARE STMT` after the `tidb_enable_prepared_plan_cache` system variable is enabled and then disabled [#49344](https://github.com/pingcap/tidb/issues/49344) @[qw4990](https://github.com/qw4990) + - Fix the issue that `LIMIT` and `OPRDERBY` might be invalid in nested `UNION` queries [#49377](https://github.com/pingcap/tidb/issues/49377) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that the `LEADING` hint does not take effect in `UNION ALL` statements [#50067](https://github.com/pingcap/tidb/issues/50067) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `COMMIT` or `ROLLBACK` operation executed through `COM_STMT_EXECUTE` fails to terminate transactions that have timed out [#49151](https://github.com/pingcap/tidb/issues/49151) @[zyguan](https://github.com/zyguan) + - Fix the issue that illegal optimizer hints might cause valid hints to be ineffective [#49308](https://github.com/pingcap/tidb/issues/49308) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that Daylight Saving Time is displayed incorrectly in some time zones [#49586](https://github.com/pingcap/tidb/issues/49586) @[overvenus](https://github.com/overvenus) + - Fix the issue that executing `SELECT INTO OUTFILE` using the `PREPARE` method incorrectly returns a success message instead of an error [#49166](https://github.com/pingcap/tidb/issues/49166) @[qw4990](https://github.com/qw4990) + - Fix the issue that TiDB might panic when performing a rolling upgrade using `tiup cluster upgrade/start` due to an interaction issue with PD [#50152](https://github.com/pingcap/tidb/issues/50152) @[zimulala](https://github.com/zimulala) + - Fix the issue that the expected optimization does not take effect when adding an index to an empty table [#49682](https://github.com/pingcap/tidb/issues/49682) @[zimulala](https://github.com/zimulala) + - Fix the issue that TiDB might OOM when a large number of tables or partitions are created [#50077](https://github.com/pingcap/tidb/issues/50077) @[zimulala](https://github.com/zimulala) + - Fix the issue that adding an index might cause inconsistent index data when the network is unstable [#49773](https://github.com/pingcap/tidb/issues/49773) @[tangenta](https://github.com/tangenta) + - Fix the execution order of DDL jobs to prevent TiCDC from receiving out-of-order DDLs [#49498](https://github.com/pingcap/tidb/issues/49498) @[tangenta](https://github.com/tangenta) + - Fix the issue that the `tidb_gogc_tuner_threshold` system variable is not adjusted accordingly after the `tidb_server_memory_limit` variable is modified [#48180](https://github.com/pingcap/tidb/issues/48180) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the query result of a range partitioned table is incorrect in some cases due to wrong partition pruning [#50082](https://github.com/pingcap/tidb/issues/50082) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that DDL operations such as renaming tables are stuck when the `CREATE TABLE` statement contains specific partitions or constraints [#50972](https://github.com/pingcap/tidb/issues/50972) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that getting the default value of a column returns an error if the column default value is dropped [#50043](https://github.com/pingcap/tidb/issues/50043) [#51324](https://github.com/pingcap/tidb/issues/51324) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that the monitoring metric `tidb_statistics_auto_analyze_total` on Grafana is not displayed as an integer [#51051](https://github.com/pingcap/tidb/issues/51051) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `tidb_merge_partition_stats_concurrency` variable does not take effect when `auto analyze` is processing partitioned tables [#47594](https://github.com/pingcap/tidb/issues/47594) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `index out of range` error might occur when a query involves JOIN operations [#42588](https://github.com/pingcap/tidb/issues/42588) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that wrong results might be returned when TiFlash late materialization processes associated columns [#49241](https://github.com/pingcap/tidb/issues/49241) [#51204](https://github.com/pingcap/tidb/issues/51204) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that parallel `Apply` might generate incorrect results when the table has a clustered index [#51372](https://github.com/pingcap/tidb/issues/51372) @[guo-shaoge](https://github.com/guo-shaoge) + ++ TiKV + + - Fix the issue that hibernated Regions are not promptly awakened in exceptional circumstances [#16368](https://github.com/tikv/tikv/issues/16368) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that the entire Region becomes unavailable when one replica is offline, by checking the last heartbeat time of all replicas of the Region before taking a node offline [#16465](https://github.com/tikv/tikv/issues/16465) @[tonyxuqqi](https://github.com/tonyxuqqi) + - Fix the issue that table properties stored in RocksDB might be inaccurate when Titan is enabled [#16319](https://github.com/tikv/tikv/issues/16319) @[hicqu](https://github.com/hicqu) + - Fix the issue that executing `tikv-ctl compact-cluster` fails when a cluster has TiFlash nodes [#16189](https://github.com/tikv/tikv/issues/16189) @[frew](https://github.com/frew) + - Fix the issue that TiKV might panic when gRPC threads are checking `is_shutdown` [#16236](https://github.com/tikv/tikv/issues/16236) @[pingyu](https://github.com/pingyu) + - Fix the issue that TiDB and TiKV might produce inconsistent results when processing `DECIMAL` arithmetic multiplication truncation [#16268](https://github.com/tikv/tikv/issues/16268) @[solotzg](https://github.com/solotzg) + - Fix the issue that `cast_duration_as_time` might return incorrect results [#16211](https://github.com/tikv/tikv/issues/16211) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that TiKV converts the time zone incorrectly for Brazil and Egypt [#16220](https://github.com/tikv/tikv/issues/16220) @[overvenus](https://github.com/overvenus) + - Fix the issue that JSON integers greater than the maximum `INT64` value but less than the maximum `UINT64` value are parsed as `FLOAT64` by TiKV, resulting in inconsistency with TiDB [#16512](https://github.com/tikv/tikv/issues/16512) @[YangKeao](https://github.com/YangKeao) + ++ PD + + - Fix the issue that slots are not fully deleted in a resource group client, which causes the number of the allocated tokens to be less than the specified value [#7346](https://github.com/tikv/pd/issues/7346) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that some TSO logs do not print the error cause [#7496](https://github.com/tikv/pd/issues/7496) @[CabinfeverB](https://github.com/CabinfeverB) + - Fix the issue that the default resource group accumulates unnecessary tokens when `BURSTABLE` is enabled [#7206](https://github.com/tikv/pd/issues/7206) @[CabinfeverB](https://github.com/CabinfeverB) + - Fix the issue that there is no output when the `evict-leader-scheduler` interface is called [#7672](https://github.com/tikv/pd/issues/7672) @[CabinfeverB](https://github.com/CabinfeverB) + - Fix the memory leak issue that occurs when `watch etcd` is not turned off correctly [#7807](https://github.com/tikv/pd/issues/7807) @[rleungx](https://github.com/rleungx) + - Fix the issue that data race occurs when the `MergeLabels` function is called [#7535](https://github.com/tikv/pd/issues/7535) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that TiDB Dashboard fails to get the TiKV profile when TLS is enabled [#7561](https://github.com/tikv/pd/issues/7561) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that the orphan peer is deleted when the number of replicas does not meet the requirements [#7584](https://github.com/tikv/pd/issues/7584) @[bufferflies](https://github.com/bufferflies) + - Fix the issue that `available_stores` is calculated incorrectly for clusters adopting the Data Replication Auto Synchronous (DR Auto-Sync) mode [#7221](https://github.com/tikv/pd/issues/7221) @[disksing](https://github.com/disksing) + - Fix the issue that `canSync` and `hasMajority` might be calculated incorrectly for clusters adopting the Data Replication Auto Synchronous (DR Auto-Sync) mode when the configuration of Placement Rules is complex [#7201](https://github.com/tikv/pd/issues/7201) @[disksing](https://github.com/disksing) + - Fix the issue that the primary AZ cannot add TiKV nodes when the secondary AZ is down for clusters adopting the Data Replication Auto Synchronous (DR Auto-Sync) mode [#7218](https://github.com/tikv/pd/issues/7218) @[disksing](https://github.com/disksing) + - Fix the issue that querying resource groups in batch might cause PD to panic [#7206](https://github.com/tikv/pd/issues/7206) @[nolouch](https://github.com/nolouch) + - Fix the issue that querying a Region without a leader using `pd-ctl` might cause PD to panic [#7630](https://github.com/tikv/pd/issues/7630) @[rleungx](https://github.com/rleungx) + - Fix the issue that the PD monitoring item `learner-peer-count` does not synchronize the old value after a leader switch [#7728](https://github.com/tikv/pd/issues/7728) @[CabinfeverB](https://github.com/CabinfeverB) + - Fix the issue that PD cannot read resource limitations when it is started with `systemd` [#7628](https://github.com/tikv/pd/issues/7628) @[bufferflies](https://github.com/bufferflies) + ++ TiFlash + + - Fix the issue that TiFlash might panic due to unstable network connections with PD during replica migration [#8323](https://github.com/pingcap/tiflash/issues/8323) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash incorrectly handles `ENUM` when the `ENUM` value is 0 [#8311](https://github.com/pingcap/tiflash/issues/8311) @[solotzg](https://github.com/solotzg) + - Fix the random invalid memory access issue that might occur with `GREATEST` or `LEAST` functions containing constant string parameters [#8604](https://github.com/pingcap/tiflash/issues/8604) @[windtalker](https://github.com/windtalker) + - Fix the issue that the `lowerUTF8` and `upperUTF8` functions do not allow characters in different cases to occupy different bytes [#8484](https://github.com/pingcap/tiflash/issues/8484) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that short queries executed successfully print excessive info logs [#8592](https://github.com/pingcap/tiflash/issues/8592) @[windtalker](https://github.com/windtalker) + - Fix the issue that the memory usage increases significantly due to slow queries [#8564](https://github.com/pingcap/tiflash/issues/8564) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that TiFlash panics after executing `ALTER TABLE ... MODIFY COLUMN ... NOT NULL`, which changes nullable columns to non-nullable [#8419](https://github.com/pingcap/tiflash/issues/8419) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that after terminating a query, TiFlash crashes due to concurrent data conflicts when a large number of tasks on TiFlash are canceled at the same time [#7432](https://github.com/pingcap/tiflash/issues/7432) @[SeaRise](https://github.com/SeaRise) + - Fix the issue that TiFlash might crash during remote reads [#8685](https://github.com/pingcap/tiflash/issues/8685) @[zanmato1984](https://github.com/zanmato1984) + - Fix the issue that TiFlash Anti Semi Join might return incorrect results when the join includes non-equivalent conditions [#8791](https://github.com/pingcap/tiflash/issues/8791) @[windtalker](https://github.com/windtalker) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that stopping a log backup task causes TiDB to crash [#50839](https://github.com/pingcap/tidb/issues/50839) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that data restore is slowed down due to absence of a leader on a TiKV node [#50566](https://github.com/pingcap/tidb/issues/50566) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that log backup gets stuck after changing the TiKV IP address on the same node [#50445](https://github.com/pingcap/tidb/issues/50445) @[3pointer](https://github.com/3pointer) + - Fix the issue that BR cannot retry when encountering an error while reading file content from S3 [#49942](https://github.com/pingcap/tidb/issues/49942) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the `Unsupported collation` error is reported when you restore data from backups of an old version [#49466](https://github.com/pingcap/tidb/issues/49466) @[3pointer](https://github.com/3pointer) + + + TiCDC + + - Fix the issue that the changefeed reports an error after `TRUNCATE PARTITION` is executed on the upstream table [#10522](https://github.com/pingcap/tiflow/issues/10522) @[sdojjy](https://github.com/sdojjy) + - Fix the issue that the changefeed `resolved ts` does not advance in extreme cases [#10157](https://github.com/pingcap/tiflow/issues/10157) @[sdojjy](https://github.com/sdojjy) + - Fix the issue that the Syncpoint table might be incorrectly replicated [#10576](https://github.com/pingcap/tiflow/issues/10576) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that after filtering out `add table partition` events is configured in `ignore-event`, TiCDC does not replicate other types of DML changes for related partitions to the downstream [#10524](https://github.com/pingcap/tiflow/issues/10524) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that the file sequence number generated by the storage service might not increment correctly when using the storage sink [#10352](https://github.com/pingcap/tiflow/issues/10352) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that TiCDC returns the `ErrChangeFeedAlreadyExists` error when concurrently creating multiple changefeeds [#10430](https://github.com/pingcap/tiflow/issues/10430) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that `snapshot lost caused by GC` is not reported in time when resuming a changefeed and the `checkpoint-ts` of the changefeed is smaller than the GC safepoint of TiDB [#10463](https://github.com/pingcap/tiflow/issues/10463) @[sdojjy](https://github.com/sdojjy) + - Fix the issue that TiCDC fails to validate `TIMESTAMP` type checksum due to time zone mismatch after data integrity validation for single-row data is enabled [#10573](https://github.com/pingcap/tiflow/issues/10573) @[3AceShowHand](https://github.com/3AceShowHand) + + + TiDB Data Migration (DM) + + - Fix the issue that a wrong binlog event type in the task configuration causes upgrade failures [#10282](https://github.com/pingcap/tiflow/issues/10282) @[GMHDBJD](https://github.com/GMHDBJD) + - Fix the issue that a table with `shard_row_id_bits` causes the schema tracker to fail to initialize [#10308](https://github.com/pingcap/tiflow/issues/10308) @[GMHDBJD](https://github.com/GMHDBJD) + + + TiDB Lightning + + - Fix the issue that TiDB Lightning reports an error when encountering invalid symbolic link files during file scanning [#49423](https://github.com/pingcap/tidb/issues/49423) @[lance6716](https://github.com/lance6716) + - Fix the issue that TiDB Lightning fails to correctly parse date values containing `0` when `NO_ZERO_IN_DATE` is not included in `sql_mode` [#50757](https://github.com/pingcap/tidb/issues/50757) @[GMHDBJD](https://github.com/GMHDBJD) diff --git a/releases/release-7.1.5.md b/releases/release-7.1.5.md new file mode 100644 index 0000000000000..525360875600b --- /dev/null +++ b/releases/release-7.1.5.md @@ -0,0 +1,119 @@ +--- +title: TiDB 7.1.5 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 7.1.5. +--- + +# TiDB 7.1.5 Release Notes + +Release date: April 26, 2024 + +TiDB version: 7.1.5 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) + +## Compatibility changes + +- Add a TiKV configuration item [`track-and-verify-wals-in-manifest`](https://docs.pingcap.com/tidb/v7.1/tikv-configuration-file#track-and-verify-wals-in-manifest-new-in-v659-and-v715) for RocksDB, which helps you investigate possible corruption of Write Ahead Log (WAL) [#16549](https://github.com/tikv/tikv/issues/16549) @[v01dstar](https://github.com/v01dstar) + +## Improvements + ++ TiDB + + - Support loading Regions in batch from PD to speed up the conversion process from the KV range to Regions when querying large tables [#51326](https://github.com/pingcap/tidb/issues/51326) @[SeaRise](https://github.com/SeaRise) + - Optimize the issue that the `ANALYZE` statement blocks the metadata lock [#47475](https://github.com/pingcap/tidb/issues/47475) @[wjhuang2016](https://github.com/wjhuang2016) + - Add a timeout mechanism for LDAP authentication to avoid the issue of resource lock (RLock) not being released in time [#51883](https://github.com/pingcap/tidb/issues/51883) @[YangKeao](https://github.com/YangKeao) + ++ TiKV + + - Add slow logs for peer and store messages [#16600](https://github.com/tikv/tikv/issues/16600) @[Connor1996](https://github.com/Connor1996) + - Avoid performing IO operations on snapshot files in raftstore threads to improve TiKV stability [#16564](https://github.com/tikv/tikv/issues/16564) @[Connor1996](https://github.com/Connor1996) + ++ PD + + - Upgrade the etcd version to v3.4.30 [#7904](https://github.com/tikv/pd/issues/7904) @[JmPotato](https://github.com/JmPotato) + ++ Tools + + + Backup & Restore (BR) + + - Support automatically abandoning log backup tasks when encountering a large checkpoint lag, to avoid prolonged blocking GC and potential cluster issues [#50803](https://github.com/pingcap/tidb/issues/50803) @[RidRisR](https://github.com/RidRisR) + - Add PITR integration test cases to cover compatibility testing for log backup and adding index acceleration [#51987](https://github.com/pingcap/tidb/issues/51987) @[Leavrth](https://github.com/Leavrth) + - Remove the invalid verification for active DDL jobs when log backup starts [#52733](https://github.com/pingcap/tidb/issues/52733) @[Leavrth](https://github.com/Leavrth) + +## Bug fixes + ++ TiDB + + - Fix the issue that querying JSON of `BINARY` type might cause an error in some cases [#51547](https://github.com/pingcap/tidb/issues/51547) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that query execution using MPP might lead to incorrect query results when a SQL statement contains `JOIN` and the `SELECT` list in the statement contains only constants [#50358](https://github.com/pingcap/tidb/issues/50358) @[yibin87](https://github.com/yibin87) + - Fix the issue that the `init-stats` process might cause TiDB to panic and the `load stats` process to quit [#51581](https://github.com/pingcap/tidb/issues/51581) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the TiDB server is marked as health before the initialization is complete [#51596](https://github.com/pingcap/tidb/issues/51596) @[shenqidebaozi](https://github.com/shenqidebaozi) + - Fix the issue that `ALTER TABLE ... COMPACT TIFLASH REPLICA` might incorrectly end when the primary key type is `VARCHAR` [#51810](https://github.com/pingcap/tidb/issues/51810) @[breezewish](https://github.com/breezewish) + - Fix the issue that TiDB crashes when `shuffleExec` quits unexpectedly [#48230](https://github.com/pingcap/tidb/issues/48230) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that the `SURVIVAL_PREFERENCES` attribute might not appear in the output of the `SHOW CREATE PLACEMENT POLICY` statement under certain conditions [#51699](https://github.com/pingcap/tidb/issues/51699) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that after the time window for automatic statistics updates is configured, statistics might still be updated outside that time window [#49552](https://github.com/pingcap/tidb/issues/49552) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that query results might be incorrect when the `HAVING` clause in a subquery contains correlated columns [#51107](https://github.com/pingcap/tidb/issues/51107) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `approx_percentile` function might cause TiDB panic [#40463](https://github.com/pingcap/tidb/issues/40463) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that the query result is incorrect when the `IN()` predicate contains `NULL` [#51560](https://github.com/pingcap/tidb/issues/51560) @[winoros](https://github.com/winoros) + - Fix the issue that the configuration file does not take effect when it contains an invalid configuration item [#51399](https://github.com/pingcap/tidb/issues/51399) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that `EXCHANGE PARTITION` incorrectly processes foreign keys [#51807](https://github.com/pingcap/tidb/issues/51807) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that querying the `TIDB_HOT_REGIONS` table might incorrectly return `INFORMATION_SCHEMA` tables [#50810](https://github.com/pingcap/tidb/issues/50810) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that the type returned by the `IFNULL` function is inconsistent with MySQL [#51765](https://github.com/pingcap/tidb/issues/51765) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that the TTL feature causes data hotspots due to incorrect data range splitting in some cases [#51527](https://github.com/pingcap/tidb/issues/51527) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that TiDB keeps sending probe requests to a TiFlash node that has been offline [#46602](https://github.com/pingcap/tidb/issues/46602) @[zyguan](https://github.com/zyguan) + - Fix the issue that AutoID Leader change might cause the value of the auto-increment column to decrease in the case of `AUTO_ID_CACHE=1` [#52600](https://github.com/pingcap/tidb/issues/52600) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that executing `INSERT IGNORE` might result in inconsistency between the unique index and the data [#51784](https://github.com/pingcap/tidb/issues/51784) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that adding a unique index might cause TiDB to panic [#52312](https://github.com/pingcap/tidb/issues/52312) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that the Window function might panic when there is a related subquery in it [#42734](https://github.com/pingcap/tidb/issues/42734) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that the `init-stats` process might cause TiDB to panic and the `load stats` process to quit [#51581](https://github.com/pingcap/tidb/issues/51581) @[hawkingrei](https://github.com/hawkingrei) + - Fix the performance regression issue caused by disabling predicate pushdown in TableDual [#50614](https://github.com/pingcap/tidb/issues/50614) @[time-and-fate](https://github.com/time-and-fate) + - Fix the issue that query results might be incorrect when the `HAVING` clause in a subquery contains correlated columns [#51107](https://github.com/pingcap/tidb/issues/51107) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `EXPLAIN` statement might display incorrect column IDs in the result when statistics for certain columns are not fully loaded [#52207](https://github.com/pingcap/tidb/issues/52207) @[time-and-fate](https://github.com/time-and-fate) + ++ TiKV + + - Fix the issue that resolve-ts is blocked when a stale Region peer ignores the GC message [#16504](https://github.com/tikv/tikv/issues/16504) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that inactive Write Ahead Logs (WALs) in RocksDB might corrupt data [#16705](https://github.com/tikv/tikv/issues/16705) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that the monitoring metric `tikv_unified_read_pool_thread_count` has no data in some cases [#16629](https://github.com/tikv/tikv/issues/16629) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that during the execution of an optimistic transaction, if other transactions initiate the resolving lock operation on it, there is a small chance that the atomicity of the transaction might be broken if the transaction's primary key has data that was previously committed in Async Commit or 1PC mode [#16620](https://github.com/tikv/tikv/issues/16620) @[MyonKeminta](https://github.com/MyonKeminta) + ++ PD + + - Fix the issue that the scheduling of write hotspots might break placement policy constraints [#7848](https://github.com/tikv/pd/issues/7848) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that the query result of `SHOW CONFIG` includes the deprecated configuration item `trace-region-flow` [#7917](https://github.com/tikv/pd/issues/7917) @[rleungx](https://github.com/rleungx) + - Fix the issue that the scaling progress is not correctly displayed [#7726](https://github.com/tikv/pd/issues/7726) @[CabinfeverB](https://github.com/CabinfeverB) + ++ TiFlash + + - Fix incorrect `local_region_num` values in logs [#8895](https://github.com/pingcap/tiflash/issues/8895) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that querying generated columns returns an error [#8787](https://github.com/pingcap/tiflash/issues/8787) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that the `ENUM` column might cause TiFlash to crash during chunk encoding [#8674](https://github.com/pingcap/tiflash/issues/8674) @[yibin87](https://github.com/yibin87) + - Fix the issue that TiFlash might panic when you insert data to columns with invalid default values in non-strict `sql_mode` [#8803](https://github.com/pingcap/tiflash/issues/8803) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that if Region migration, split, or merge occurs after the precision of a `TIME` column is modified, queries might fail [#8601](https://github.com/pingcap/tiflash/issues/8601) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that removing a log backup task after it is paused does not immediately restore the GC safepoint [#52082](https://github.com/pingcap/tidb/issues/52082) @[3pointer](https://github.com/3pointer) + - Fix the issue that too many logs are printed when a full backup fails [#51572](https://github.com/pingcap/tidb/issues/51572) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that BR could not back up the `AUTO_RANDOM` ID allocation progress if the `AUTO_RANDOM` column is in a composite clustered index [#52255](https://github.com/pingcap/tidb/issues/52255) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that TiKV panics when a full backup fails to find a peer in some extreme cases [#16394](https://github.com/tikv/tikv/issues/16394) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that a PD connection failure could cause the TiDB instance where the log backup advancer owner is located to panic [#52597](https://github.com/pingcap/tidb/issues/52597) @[YuJuncen](https://github.com/YuJuncen) + - Fix an unstable test case [#52547](https://github.com/pingcap/tidb/issues/52547) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the global checkpoint of log backup is advanced ahead of the actual backup file write point due to TiKV restart, which might cause a small amount of backup data loss [#16809](https://github.com/tikv/tikv/issues/16809) @[YuJuncen](https://github.com/YuJuncen) + - Fix a rare issue that special event timing might cause the data loss in log backup [#16739](https://github.com/tikv/tikv/issues/16739) @[YuJuncen](https://github.com/YuJuncen) + + + TiCDC + + - Fix the issue that TiCDC fails to execute the `Exchange Partition ... With Validation` DDL downstream after it is written upstream, causing the changefeed to get stuck [#10859](https://github.com/pingcap/tiflow/issues/10859) @[hongyunyan](https://github.com/hongyunyan) + - Fix the issue that `snapshot lost caused by GC` is not reported in time when resuming a changefeed and the `checkpoint-ts` of the changefeed is smaller than the GC safepoint of TiDB [#10463](https://github.com/pingcap/tiflow/issues/10463) @[sdojjy](https://github.com/sdojjy) + - Fix the issue that TiCDC panics when scheduling table replication tasks [#10613](https://github.com/pingcap/tiflow/issues/10613) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that data is written to a wrong CSV file due to wrong BarrierTS in scenarios where DDL statements are executed frequently [#10668](https://github.com/pingcap/tiflow/issues/10668) @[lidezhu](https://github.com/lidezhu) + - Fix the issue that a changefeed with eventual consistency enabled might fail when the object storage sink encounters a temporary failure [#10710](https://github.com/pingcap/tiflow/issues/10710) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that the old value part of `open-protocol` incorrectly outputs the default value according to the `STRING` type instead of its actual type [#10803](https://github.com/pingcap/tiflow/issues/10803) @[3AceShowHand](https://github.com/3AceShowHand) + + + TiDB Lightning + + - Fix the issue that TiDB Lightning panics when importing an empty table of Parquet format [#52518](https://github.com/pingcap/tidb/issues/52518) @[kennytm](https://github.com/kennytm) + - Fix the issue that sensitive information in logs is printed in server mode [#36374](https://github.com/pingcap/tidb/issues/36374) @[kennytm](https://github.com/kennytm) diff --git a/releases/release-7.1.6.md b/releases/release-7.1.6.md new file mode 100644 index 0000000000000..1e4f2c030b932 --- /dev/null +++ b/releases/release-7.1.6.md @@ -0,0 +1,303 @@ +--- +title: TiDB 7.1.6 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 7.1.6. +--- + +# TiDB 7.1.6 Release Notes + +Release date: November 21, 2024 + +TiDB version: 7.1.6 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) + +## Compatibility changes + +- Set a default limit of 2048 for DDL historical tasks retrieved through the [TiDB HTTP API](https://github.com/pingcap/tidb/blob/release-7.1/docs/tidb_http_api.md) to prevent OOM issues caused by excessive historical tasks [#55711](https://github.com/pingcap/tidb/issues/55711) @[joccau](https://github.com/joccau) +- In earlier versions, when processing a transaction containing `UPDATE` changes, if the primary key or non-null unique index value is modified in an `UPDATE` event, TiCDC splits this event into `DELETE` and `INSERT` events. Starting from v7.1.6, when using the MySQL sink, TiCDC splits an `UPDATE` event into `DELETE` and `INSERT` events if the transaction `commitTS` for the `UPDATE` change is less than TiCDC `thresholdTS` (which is the current timestamp fetched from PD when TiCDC starts replicating the corresponding table to the downstream). This behavior change addresses the issue of downstream data inconsistencies caused by the potentially incorrect order of `UPDATE` events received by TiCDC, which can lead to an incorrect order of split `DELETE` and `INSERT` events. For more information, see [documentation](https://docs.pingcap.com/tidb/v7.1/ticdc-split-update-behavior#split-update-events-for-mysql-sinks). [#10918](https://github.com/pingcap/tiflow/issues/10918) @[lidezhu](https://github.com/lidezhu) +- Must set the line terminator when using TiDB Lightning `strict-format` to import CSV files [#37338](https://github.com/pingcap/tidb/issues/37338) @[lance6716](https://github.com/lance6716) +- Change the scope of the TiKV configuration item [`server.grpc-compression-type`](/tikv-configuration-file.md#grpc-compression-type): + + - In v7.1.x versions earlier than v7.1.6, this configuration item only affects the compression algorithm of gRPC messages between TiKV nodes. + - Starting from v7.1.6, this configuration item also affects the compression algorithm of gRPC response messages sent from TiKV to TiDB. Enabling compression might consume more CPU resources. [#17176](https://github.com/tikv/tikv/issues/17176) @[ekexium](https://github.com/ekexium) + +## Improvements + ++ TiDB + + - Adjust estimation results from 0 to 1 for equality conditions that do not hit TopN when statistics are entirely composed of TopN and the modified row count in the corresponding table statistics is non-zero [#47400](https://github.com/pingcap/tidb/issues/47400) @[terry1purcell](https://github.com/terry1purcell) + - Remove stores without Regions during MPP load balancing [#52313](https://github.com/pingcap/tidb/issues/52313) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Improve the MySQL compatibility of expression default values displayed in the output of `SHOW CREATE TABLE` [#52939](https://github.com/pingcap/tidb/issues/52939) @[CbcWestwolf](https://github.com/CbcWestwolf) + - By batch deleting TiFlash placement rules, improve the processing speed of data GC after performing the `TRUNCATE` or `DROP` operation on partitioned tables [#54068](https://github.com/pingcap/tidb/issues/54068) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Improve sync load performance to reduce latency in loading statistics [#52294](https://github.com/pingcap/tidb/issues/52294) [hawkingrei](https://github.com/hawkingrei) + ++ TiKV + + - Add slow logs for peer and store messages [#16600](https://github.com/tikv/tikv/issues/16600) @[Connor1996](https://github.com/Connor1996) + - Optimize the compaction trigger mechanism of RocksDB to accelerate disk space reclamation when handling a large number of DELETE versions [#17269](https://github.com/tikv/tikv/issues/17269) @[AndreMouche](https://github.com/AndreMouche) + - Optimize the jittery access delay when restarting TiKV due to waiting for the log to be applied, improving the stability of TiKV [#15874](https://github.com/tikv/tikv/issues/15874) @[LykxSassinator](https://github.com/LykxSassinator) + - Remove unnecessary async blocks to reduce memory usage [#16540](https://github.com/tikv/tikv/issues/16540) @[overvenus](https://github.com/overvenus) + ++ TiFlash + + - Optimize the execution efficiency of `LENGTH()` and `ASCII()` functions [#9344](https://github.com/pingcap/tiflash/issues/9344) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Mitigate the issue that TiFlash might panic due to updating certificates after TLS is enabled [#8535](https://github.com/pingcap/tiflash/issues/8535) @[windtalker](https://github.com/windtalker) + - Improve the cancel mechanism of the JOIN operator, so that the JOIN operator can respond to cancel requests in a timely manner [#9430](https://github.com/pingcap/tiflash/issues/9430) @[windtalker](https://github.com/windtalker) + - Reduce lock conflicts under highly concurrent data read operations and optimize short query performance [#9125](https://github.com/pingcap/tiflash/issues/9125) @[JinheLin](https://github.com/JinheLin) + - Improve the garbage collection speed of outdated data in the background for tables with clustered indexes [#9529](https://github.com/pingcap/tiflash/issues/9529) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - Enhance the tolerance of log backup to merge operations. When encountering a reasonably long merge operation, log backup tasks are less likely to enter the error state [#16554](https://github.com/tikv/tikv/issues/16554) @[YuJuncen](https://github.com/YuJuncen) + - BR cleans up empty SST files during data recovery [#16005](https://github.com/tikv/tikv/issues/16005) @[Leavrth](https://github.com/Leavrth) + - Increase the number of retries for failures caused by DNS errors [#53029](https://github.com/pingcap/tidb/issues/53029) @[YuJuncen](https://github.com/YuJuncen) + - Increase the number of retries for failures caused by the absence of a leader in a Region [#54017](https://github.com/pingcap/tidb/issues/54017) @[Leavrth](https://github.com/Leavrth) + - Except for the `br log restore` subcommand, all other `br log` subcommands support skipping the loading of the TiDB `domain` data structure to reduce memory consumption [#52088](https://github.com/pingcap/tidb/issues/52088) @[Leavrth](https://github.com/Leavrth) + - Support checking whether the disk space in TiKV is sufficient before TiKV downloads each SST file. If the space is insufficient, BR terminates the restore and returns an error [#17224](https://github.com/tikv/tikv/issues/17224) @[RidRisR](https://github.com/RidRisR) + - Support setting Alibaba Cloud access credentials through environment variables [#45551](https://github.com/pingcap/tidb/issues/45551) @[RidRisR](https://github.com/RidRisR) + - Reduce unnecessary log printing during backup [#55902](https://github.com/pingcap/tidb/issues/55902) @[Leavrth](https://github.com/Leavrth) + + + TiCDC + + - Support directly outputting raw events when the downstream is a Message Queue (MQ) or cloud storage [#11211](https://github.com/pingcap/tiflow/issues/11211) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Improve memory stability during data recovery using redo logs to reduce the probability of OOM [#10900](https://github.com/pingcap/tiflow/issues/10900) @[CharlesCheung96](https://github.com/CharlesCheung96) + - When the downstream is TiDB with the `SUPER` permission granted, TiCDC supports querying the execution status of `ADD INDEX DDL` from the downstream database to avoid data replication failure due to timeout in retrying executing the DDL statement in some cases [#10682](https://github.com/pingcap/tiflow/issues/10682) @[CharlesCheung96](https://github.com/CharlesCheung96) + + + TiDB Data Migration (DM) + + - Upgrade `go-mysql` to 1.9.1 to support connecting to MySQL server 8.0 using passwords longer than 19 characters [#11603](https://github.com/pingcap/tiflow/pull/11603) @[fishiu](https://github.com/fishiu) + +## Bug fixes + ++ TiDB + + - Fix the issue of inconsistent data indexes caused by concurrent DML operations when adding a unique index [#52914](https://github.com/pingcap/tidb/issues/52914) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that comparing a column of `YEAR` type with an unsigned integer that is out of range causes incorrect results [#50235](https://github.com/pingcap/tidb/issues/50235) @[qw4990](https://github.com/qw4990) + - Fix the issue that `INDEX_HASH_JOIN` cannot exit properly when SQL is abnormally interrupted [#54688](https://github.com/pingcap/tidb/issues/54688) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that the network partition during adding indexes using the Distributed eXecution Framework (DXF) might cause inconsistent data indexes [#54897](https://github.com/pingcap/tidb/issues/54897) @[tangenta](https://github.com/tangenta) + - Fix the issue that using `SHOW WARNINGS;` to obtain warnings might cause a panic [#48756](https://github.com/pingcap/tidb/issues/48756) @[xhebox](https://github.com/xhebox) + - Fix the issue that querying the `INFORMATION_SCHEMA.CLUSTER_SLOW_QUERY` table might cause TiDB to panic [#54324](https://github.com/pingcap/tidb/issues/54324) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue of abnormally high memory usage caused by `memTracker` not being detached when the `HashJoin` or `IndexLookUp` operator is the driven side sub-node of the `Apply` operator [#54005](https://github.com/pingcap/tidb/issues/54005) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that recursive CTE queries might result in invalid pointers [#54449](https://github.com/pingcap/tidb/issues/54449) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that an empty projection causes TiDB to panic [#49109](https://github.com/pingcap/tidb/issues/49109) @[winoros](https://github.com/winoros) + - Fix the issue that TiDB might return incorrect query results when you query tables with virtual columns in transactions that involve data modification operations [#53951](https://github.com/pingcap/tidb/issues/53951) @[qw4990](https://github.com/qw4990) + - Fix the issue that for tables containing auto-increment columns with `AUTO_ID_CACHE=1`, setting `auto_increment_increment` and `auto_increment_offset` system variables to non-default values might cause incorrect auto-increment ID allocation [#52622](https://github.com/pingcap/tidb/issues/52622) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that subqueries included in the `ALL` function might cause incorrect results [#52755](https://github.com/pingcap/tidb/issues/52755) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that predicates cannot be pushed down properly when the filter condition of a SQL query contains virtual columns and the execution condition contains `UnionScan` [#54870](https://github.com/pingcap/tidb/issues/54870) @[qw4990](https://github.com/qw4990) + - Fix the issue that subqueries in an `UPDATE` list might cause TiDB to panic [#52687](https://github.com/pingcap/tidb/issues/52687) @[winoros](https://github.com/winoros) + - Fix the issue that indirect placeholder `?` references in a `GROUP BY` statement cannot find columns [#53872](https://github.com/pingcap/tidb/issues/53872) @[qw4990](https://github.com/qw4990) + - Fix the issue that disk files might not be deleted after the `Sort` operator spills and a query error occurs [#55061](https://github.com/pingcap/tidb/issues/55061) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue of reusing wrong point get plans for `SELECT ... FOR UPDATE` [#54652](https://github.com/pingcap/tidb/issues/54652) @[qw4990](https://github.com/qw4990) + - Fix the issue that `max_execute_time` settings at multiple levels interfere with each other [#50914](https://github.com/pingcap/tidb/issues/50914) @[jiyfhust](https://github.com/jiyfhust) + - Fix the issue that the histogram and TopN in the primary key column statistics are not loaded after restarting TiDB [#37548](https://github.com/pingcap/tidb/issues/37548) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the TopN operator might be pushed down incorrectly [#37986](https://github.com/pingcap/tidb/issues/37986) @[qw4990](https://github.com/qw4990) + - Fix the issue that the performance of the `SELECT ... WHERE ... ORDER BY ...` statement execution is poor in some cases [#54969](https://github.com/pingcap/tidb/issues/54969) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that TiDB reports an error in the log when closing the connection in some cases [#53689](https://github.com/pingcap/tidb/issues/53689) @[jackysp](https://github.com/jackysp) + - Fix the issue that the illegal column type `DECIMAL(0,0)` can be created in some cases [#53779](https://github.com/pingcap/tidb/issues/53779) @[tangenta](https://github.com/tangenta) + - Fix the issue that obtaining the column information using `information_schema.columns` returns warning 1356 when a subquery is used as a column definition in a view definition [#54343](https://github.com/pingcap/tidb/issues/54343) @[lance6716](https://github.com/lance6716) + - Fix the issue that the optimizer incorrectly estimates the number of rows as 1 when accessing a unique index with the query condition `column IS NULL` [#56116](https://github.com/pingcap/tidb/issues/56116) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that `SELECT INTO OUTFILE` does not work when clustered indexes are used as predicates [#42093](https://github.com/pingcap/tidb/issues/42093) @[qw4990](https://github.com/qw4990) + - Fix the issue of incorrect WARNINGS information when using Optimizer Hints [#53767](https://github.com/pingcap/tidb/issues/53767) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the Sync Load QPS monitoring metric is incorrect [#53558](https://github.com/pingcap/tidb/issues/53558) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that executing `CREATE OR REPLACE VIEW` concurrently might result in the `table doesn't exist` error [#53673](https://github.com/pingcap/tidb/issues/53673) @[tangenta](https://github.com/tangenta) + - Fix the issue that restoring a table with `AUTO_ID_CACHE=1` using the `RESTORE` statement might cause a `Duplicate entry` error [#52680](https://github.com/pingcap/tidb/issues/52680) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the `SUB_PART` value in the `INFORMATION_SCHEMA.STATISTICS` table is `NULL` [#55812](https://github.com/pingcap/tidb/issues/55812) @[Defined2014](https://github.com/Defined2014) + - Fix the overflow issue of the `Longlong` type in predicates [#45783](https://github.com/pingcap/tidb/issues/45783) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that incorrect results are returned when the cached execution plans contain the comparison between date types and `unix_timestamp` [#48165](https://github.com/pingcap/tidb/issues/48165) @[qw4990](https://github.com/qw4990) + - Fix the issue that the `LENGTH()` condition is unexpectedly removed when the collation is `utf8_bin` or `utf8mb4_bin` [#53730](https://github.com/pingcap/tidb/issues/53730) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that when an `UPDATE` or `DELETE` statement contains a recursive CTE, the statement might report an error or not take effect [#55666](https://github.com/pingcap/tidb/issues/55666) @[time-and-fate](https://github.com/time-and-fate) + - Fix the issue that TiDB might hang or return incorrect results when executing a query containing a correlated subquery and CTE [#55551](https://github.com/pingcap/tidb/issues/55551) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that statistics for string columns with non-binary collations might fail to load when initializing statistics [#55684](https://github.com/pingcap/tidb/issues/55684) @[winoros](https://github.com/winoros) + - Fix the issue that IndexJoin produces duplicate rows when calculating hash values in the Left Outer Anti Semi type [#52902](https://github.com/pingcap/tidb/issues/52902) @[yibin87](https://github.com/yibin87) + - Fix the issue that a query statement that contains `UNION` might return incorrect results [#52985](https://github.com/pingcap/tidb/issues/52985) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that empty `groupOffset` in `StreamAggExec` might cause TiDB to panic [#53867](https://github.com/pingcap/tidb/issues/53867) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that RANGE partitioned tables that are not strictly self-incrementing can be created [#54829](https://github.com/pingcap/tidb/issues/54829) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that the query might get stuck when terminated because the memory usage exceeds the limit set by `tidb_mem_quota_query` [#55042](https://github.com/pingcap/tidb/issues/55042) @[yibin87](https://github.com/yibin87) + - Fix the issue that the `STATE` field in the `INFORMATION_SCHEMA.TIDB_TRX` table is empty due to the `size` of the `STATE` field not being defined [#53026](https://github.com/pingcap/tidb/issues/53026) @[cfzjywxk](https://github.com/cfzjywxk) + - Fix the data race issue in `IndexNestedLoopHashJoin` [#49692](https://github.com/pingcap/tidb/issues/49692) @[solotzg](https://github.com/solotzg) + - Fix the issue that a wrong TableDual plan causes empty query results [#50051](https://github.com/pingcap/tidb/issues/50051) @[onlyacat](https://github.com/onlyacat) + - Fix the issue that the `tot_col_size` column in the `mysql.stats_histograms` table might be a negative number [#55126](https://github.com/pingcap/tidb/issues/55126) @[qw4990](https://github.com/qw4990) + - Fix the issue that data conversion from the `FLOAT` type to the `UNSIGNED` type returns incorrect results [#41736](https://github.com/pingcap/tidb/issues/41736) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that TiDB fails to reject unauthenticated user connections in some cases when using the `auth_socket` authentication plugin [#54031](https://github.com/pingcap/tidb/issues/54031) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that the `memory_quota` hint might not work in subqueries [#53834](https://github.com/pingcap/tidb/issues/53834) @[qw4990](https://github.com/qw4990) + - Fix the issue that the metadata lock fails to prevent DDL operations from executing in the plan cache scenario [#51407](https://github.com/pingcap/tidb/issues/51407) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that using `CURRENT_DATE()` as the default value for a column results in incorrect query results [#53746](https://github.com/pingcap/tidb/issues/53746) @[tangenta](https://github.com/tangenta) + - Fix the issue that the `COALESCE()` function returns incorrect result type for `DATE` type parameters [#46475](https://github.com/pingcap/tidb/issues/46475) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Reset the parameters in the `Open` method of `PipelinedWindow` to fix the unexpected error that occurs when the `PipelinedWindow` is used as a child node of `Apply` due to the reuse of previous parameter values caused by repeated opening and closing operations [#53600](https://github.com/pingcap/tidb/issues/53600) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the incorrect result of the TopN operator in correlated subqueries [#52777](https://github.com/pingcap/tidb/issues/52777) @[yibin87](https://github.com/yibin87) + - Fix the issue that the recursive CTE operator incorrectly tracks memory usage [#54181](https://github.com/pingcap/tidb/issues/54181) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that an error occurs when using `SHOW COLUMNS` to view columns in a view [#54964](https://github.com/pingcap/tidb/issues/54964) @[lance6716](https://github.com/lance6716) + - Fix the issue that reducing the value of `tidb_ttl_delete_worker_count` during TTL job execution makes the job fail to complete [#55561](https://github.com/pingcap/tidb/issues/55561) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that using a view does not work in recursive CTE [#49721](https://github.com/pingcap/tidb/issues/49721) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that TiDB does not create corresponding statistics metadata (`stats_meta`) when creating a table with foreign keys [#53652](https://github.com/pingcap/tidb/issues/53652) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the query might return incorrect results instead of an error after being killed [#50089](https://github.com/pingcap/tidb/issues/50089) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that the statistics synchronous loading mechanism might fail unexpectedly under high query concurrency [#52294](https://github.com/pingcap/tidb/issues/52294) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that certain filter conditions in queries might cause the planner module to report an `invalid memory address or nil pointer dereference` error [#53582](https://github.com/pingcap/tidb/issues/53582) [#53580](https://github.com/pingcap/tidb/issues/53580) [#53594](https://github.com/pingcap/tidb/issues/53594) [#53603](https://github.com/pingcap/tidb/issues/53603) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that the TiDB synchronously loading statistics mechanism retries to load empty statistics indefinitely and prints the `fail to get stats version for this histogram` log [#52657](https://github.com/pingcap/tidb/issues/52657) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `TIMESTAMPADD()` function goes into an infinite loop when the first argument is `month` and the second argument is negative [#54908](https://github.com/pingcap/tidb/issues/54908) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that TiDB might crash when `tidb_mem_quota_analyze` is enabled and the memory used by updating statistics exceeds the limit [#52601](https://github.com/pingcap/tidb/issues/52601) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that `duplicate entry` might occur when adding unique indexes [#56161](https://github.com/pingcap/tidb/issues/56161) @[tangenta](https://github.com/tangenta) + - Fix the issue that the query latency of stale reads increases, caused by information schema cache misses [#53428](https://github.com/pingcap/tidb/issues/53428) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that the `Distinct_count` information in GlobalStats might be incorrect [#53752](https://github.com/pingcap/tidb/issues/53752) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that executing the `SELECT DISTINCT CAST(col AS DECIMAL), CAST(col AS SIGNED) FROM ...` query might return incorrect results [#53726](https://github.com/pingcap/tidb/issues/53726) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `read_from_storage` hint might not take effect when the query has an available Index Merge execution plan [#56217](https://github.com/pingcap/tidb/issues/56217) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that the `TIMESTAMPADD()` function returns incorrect results [#41052](https://github.com/pingcap/tidb/issues/41052) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that `PREPARE`/`EXECUTE` statements with the `CONV` expression containing a `?` argument might result in incorrect query results when executed multiple times [#53505](https://github.com/pingcap/tidb/issues/53505) @[qw4990](https://github.com/qw4990) + - Fix the issue that the memory used by transactions might be tracked multiple times [#53984](https://github.com/pingcap/tidb/issues/53984) @[ekexium](https://github.com/ekexium) + - Fix the issue that column pruning without using shallow copies of slices might cause TiDB to panic [#52768](https://github.com/pingcap/tidb/issues/52768) @[winoros](https://github.com/winoros) + - Fix the issue that a SQL binding containing window functions might not take effect in some cases [#55981](https://github.com/pingcap/tidb/issues/55981) @[winoros](https://github.com/winoros) + - Fix the issue that TiDB might panic when parsing index data [#47115](https://github.com/pingcap/tidb/issues/47115) @[zyguan](https://github.com/zyguan) + - Fix the issue that TiDB might report an error due to GC when loading statistics at startup [#53592](https://github.com/pingcap/tidb/issues/53592) @[you06](https://github.com/you06) + - Fix the issue that an error occurs when a DML statement contains nested generated columns [#53967](https://github.com/pingcap/tidb/issues/53967) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that TiDB panics when executing the `SHOW ERRORS` statement with a predicate that is always `true` [#46962](https://github.com/pingcap/tidb/issues/46962) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that improper use of metadata locks might lead to writing anomalous data when using the plan cache under certain circumstances [#53634](https://github.com/pingcap/tidb/issues/53634) @[zimulala](https://github.com/zimulala) + - Fix the issue of data index inconsistency caused by retries during index addition [#55808](https://github.com/pingcap/tidb/issues/55808) @[lance6716](https://github.com/lance6716) + - Fix the issue that unstable unique IDs of columns might cause the `UPDATE` statement to return errors [#53236](https://github.com/pingcap/tidb/issues/53236) @[winoros](https://github.com/winoros) + - Fix the issue that after a statement within a transaction is killed by OOM, if TiDB continues to execute the next statement within the same transaction, you might get an error `Trying to start aggressive locking while it's already started` and a panic occurs [#53540](https://github.com/pingcap/tidb/issues/53540) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that executing `RECOVER TABLE BY JOB JOB_ID;` might cause TiDB to panic [#55113](https://github.com/pingcap/tidb/issues/55113) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that executing `ADD INDEX` might fail after modifying the PD member in the distributed execution framework [#48680](https://github.com/pingcap/tidb/issues/48680) @[lance6716](https://github.com/lance6716) + - Fix the issue that two DDL Owners might exist at the same time [#54689](https://github.com/pingcap/tidb/issues/54689) @[joccau](https://github.com/joccau) + - Fix the issue that TiDB rolling restart during the execution of `ADD INDEX` might cause the adding index operation to fail [#52805](https://github.com/pingcap/tidb/issues/52805) @[tangenta](https://github.com/tangenta) + - Fix the issue that the `LOAD DATA ... REPLACE INTO` operation causes data inconsistency [#56408](https://github.com/pingcap/tidb/issues/56408) @[fzzf678](https://github.com/fzzf678) + - Fix the issue that the `AUTO_INCREMENT` field is not correctly set after importing data using the `IMPORT INTO` statement [#56476](https://github.com/pingcap/tidb/issues/56476) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that TiDB does not check for the existence of local files before restoring from a checkpoint [#53009](https://github.com/pingcap/tidb/issues/53009) @[lance6716](https://github.com/lance6716) + - Fix the issue that the DM schema tracker cannot create indexes longer than the default length [#55138](https://github.com/pingcap/tidb/issues/55138) @[lance6716](https://github.com/lance6716) + - Fix the issue that `ALTER TABLE` does not handle the `AUTO_INCREMENT` field correctly [#47899](https://github.com/pingcap/tidb/issues/47899) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that unreleased session resources might lead to memory leaks [#56271](https://github.com/pingcap/tidb/issues/56271) @[lance6716](https://github.com/lance6716) + - Fix the issue that float or integer overflow affects the plan cache [#46538](https://github.com/pingcap/tidb/issues/46538) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that part of the memory of the `IndexLookUp` operator is not tracked [#56440](https://github.com/pingcap/tidb/issues/56440) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that stale read does not strictly verify the timestamp of the read operation, resulting in a small probability of affecting the consistency of the transaction when an offset exists between the TSO and the real physical time [#56809](https://github.com/pingcap/tidb/issues/56809) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that TTL might fail if TiKV is not selected as the storage engine [#56402](https://github.com/pingcap/tidb/issues/56402) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that TTL tasks cannot be canceled when there is a write conflict [#56422](https://github.com/pingcap/tidb/issues/56422) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that inserting oversized numbers in scientific notation causes an error `ERROR 1264 (22003)`, to make the behavior consistent with MySQL [#47787](https://github.com/pingcap/tidb/issues/47787) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that when canceling a TTL task, the corresponding SQL is not killed forcibly [#56511](https://github.com/pingcap/tidb/issues/56511) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that the `INSERT ... ON DUPLICATE KEY` statement is not compatible with `mysql_insert_id` [#55965](https://github.com/pingcap/tidb/issues/55965) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that audit log filtering does not take effect when SQL cannot build an execution plan [#50988](https://github.com/pingcap/tidb/issues/50988) @[CbcWestwolf](https://github.com/CbcWestwolf) + - Fix the issue that existing TTL tasks are executed unexpectedly frequently in a cluster that is upgraded from v6.5 to v7.5 or later [#56539](https://github.com/pingcap/tidb/issues/56539) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that the `CAST` function does not support explicitly setting the character set [#55677](https://github.com/pingcap/tidb/issues/55677) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that TiDB does not check the index length limitation when executing `ADD INDEX` [#56930](https://github.com/pingcap/tidb/issues/56930) @[fzzf678](https://github.com/fzzf678) + ++ TiKV + + - Add the `RawKvMaxTimestampNotSynced` error, log detailed error information in `errorpb.Error.max_ts_not_synced`, and add a retry mechanism for the `must_raw_put` operation when this error occurs [#16789](https://github.com/tikv/tikv/issues/16789) @[pingyu](https://github.com/pingyu) + - Fix a traffic control issue that might occur after deleting large tables or partitions [#17304](https://github.com/tikv/tikv/issues/17304) @[SpadeA-Tang](https://github.com/SpadeA-Tang) + - Fix the panic issue that occurs when read threads access outdated indexes in the MemTable of the Raft Engine [#17383](https://github.com/tikv/tikv/issues/17383) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that CDC and log-backup do not limit the timeout of `check_leader` using the `advance-ts-interval` configuration, causing the `resolved_ts` lag to be too large when TiKV restarts normally in some cases [#17107](https://github.com/tikv/tikv/issues/17107) @[SpadeA-Tang](https://github.com/SpadeA-Tang) + - Fix the issue that SST files imported by TiDB Lightning are lost after TiKV is restarted [#15912](https://github.com/tikv/tikv/issues/15912) @[lance6716](https://github.com/lance6716) + - Fix the issue that TiKV might panic due to ingesting deleted `sst_importer` SST files [#15053](https://github.com/tikv/tikv/issues/15053) @[lance6716](https://github.com/lance6716) + - Fix the issue that when there are a large number of Regions in a TiKV instance, TiKV might be OOM during data import [#16229](https://github.com/tikv/tikv/issues/16229) @[SpadeA-Tang](https://github.com/SpadeA-Tang) + - Fix the issue that bloom filters are incompatible between earlier versions (earlier than v7.1) and later versions [#17272](https://github.com/tikv/tikv/issues/17272) @[v01dstar](https://github.com/v01dstar) + - Fix the issue that setting the gRPC message compression method via `grpc-compression-type` does not take effect on messages sent from TiKV to TiDB [#17176](https://github.com/tikv/tikv/issues/17176) @[ekexium](https://github.com/ekexium) + - Fix the issue of unstable test cases, ensuring that each test uses an independent temporary directory to avoid online configuration changes affecting other test cases [#16871](https://github.com/tikv/tikv/issues/16871) @[glorv](https://github.com/glorv) + - Fix the issue that when a large number of transactions are queuing for lock release on the same key and the key is frequently updated, excessive pressure on deadlock detection might cause TiKV OOM issues [#17394](https://github.com/tikv/tikv/issues/17394) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that the decimal part of the `DECIMAL` type is incorrect in some cases [#16913](https://github.com/tikv/tikv/issues/16913) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that the `CONV()` function in queries might overflow during numeric system conversion, leading to TiKV panic [#16969](https://github.com/tikv/tikv/issues/16969) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that TiKV might panic when a stale replica processes Raft snapshots, triggered by a slow split operation and immediate removal of the new replica [#17469](https://github.com/tikv/tikv/issues/17469) @[hbisheng](https://github.com/hbisheng) + - Fix the issue that highly concurrent Coprocessor requests might cause TiKV OOM [#16653](https://github.com/tikv/tikv/issues/16653) @[overvenus](https://github.com/overvenus) + - Fix the issue that prevents master key rotation when the master key is stored in a Key Management Service (KMS) [#17410](https://github.com/tikv/tikv/issues/17410) @[hhwyt](https://github.com/hhwyt) + - Fix the issue that the output of the `raft region` command in tikv-ctl does not include the Region status information [#17037](https://github.com/tikv/tikv/issues/17037) @[glorv](https://github.com/glorv) + - Fix the issue that the **Storage async write duration** monitoring metric on the TiKV panel in Grafana is inaccurate [#17579](https://github.com/tikv/tikv/issues/17579) @[overvenus](https://github.com/overvenus) + - Fix the issue that TiKV converts the time zone incorrectly for Brazil and Egypt [#16220](https://github.com/tikv/tikv/issues/16220) @[overvenus](https://github.com/overvenus) + ++ PD + + - Fix the memory leak issue in label statistics [#8700](https://github.com/tikv/pd/issues/8700) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that resource groups print excessive logs [#8159](https://github.com/tikv/pd/issues/8159) @[nolouch](https://github.com/nolouch) + - Fix the performance jitter issue caused by frequent creation of random number generator [#8674](https://github.com/tikv/pd/issues/8674) @[rleungx](https://github.com/rleungx) + - Fix the memory leak issue in Region statistics [#8710](https://github.com/tikv/pd/issues/8710) @[rleungx](https://github.com/rleungx) + - Fix the memory leak issue in hotspot cache [#8698](https://github.com/tikv/pd/issues/8698) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that `evict-leader-scheduler` fails to work properly when it is repeatedly created with the same Store ID [#8756](https://github.com/tikv/pd/issues/8756) @[okJiang](https://github.com/okJiang) + - Fix the issue that setting `replication.strictly-match-label` to `true` causes TiFlash to fail to start [#8480](https://github.com/tikv/pd/issues/8480) @[rleungx](https://github.com/rleungx) + - Fix the issue that changing the log level via the configuration file does not take effect [#8117](https://github.com/tikv/pd/issues/8117) @[rleungx](https://github.com/rleungx) + - Fix the issue that resource groups could not effectively limit resource usage under high concurrency [#8435](https://github.com/tikv/pd/issues/8435) @[nolouch](https://github.com/nolouch) + - Fix the data race issue that PD encounters during operator checks [#8263](https://github.com/tikv/pd/issues/8263) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that a resource group encounters quota limits when requesting tokens for more than 500 ms [#8349](https://github.com/tikv/pd/issues/8349) @[nolouch](https://github.com/nolouch) + - Fix the issue that some logs are not redacted [#8419](https://github.com/tikv/pd/issues/8419) @[rleungx](https://github.com/rleungx) + - Fix the issue that no error is reported when binding a role to a resource group [#54417](https://github.com/pingcap/tidb/issues/54417) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that PD's Region API cannot be requested when a large number of Regions exist [#55872](https://github.com/pingcap/tidb/issues/55872) @[rleungx](https://github.com/rleungx) + - Fix the issue that a large number of retries occur when canceling resource groups queries [#8217](https://github.com/tikv/pd/issues/8217) @[nolouch](https://github.com/nolouch) + - Fix the issue that the encryption manager is not initialized before use [#8384](https://github.com/tikv/pd/issues/8384) @[rleungx](https://github.com/rleungx) + - Fix the issue that the `Filter target` monitoring metric for PD does not provide scatter range information [#8125](https://github.com/tikv/pd/issues/8125) @[HuSharp](https://github.com/HuSharp) + - Fix the data race issue of resource groups [#8267](https://github.com/tikv/pd/issues/8267) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that setting the TiKV configuration item [`coprocessor.region-split-size`](/tikv-configuration-file.md#region-split-size) to a value less than 1 MiB causes PD panic [#8323](https://github.com/tikv/pd/issues/8323) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that when using a wrong parameter in `evict-leader-scheduler`, PD does not report errors correctly and some schedulers are unavailable [#8619](https://github.com/tikv/pd/issues/8619) @[rleungx](https://github.com/rleungx) + - Fix the issue that slots are not fully deleted in a resource group client, which causes the number of the allocated tokens to be less than the specified value [#7346](https://github.com/tikv/pd/issues/7346) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that down peers might not recover when using Placement Rules [#7808](https://github.com/tikv/pd/issues/7808) @[rleungx](https://github.com/rleungx) + ++ TiFlash + + - Fix the issue that TiFlash metadata might become corrupted and cause the process to panic when upgrading a cluster from a version earlier than v6.5.0 to v6.5.0 or later [#9039](https://github.com/pingcap/tiflash/issues/9039) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that some queries might report a column type mismatch error after late materialization is enabled [#9175](https://github.com/pingcap/tiflash/issues/9175) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that some queries might report errors when late materialization is enabled [#9472](https://github.com/pingcap/tiflash/issues/9472) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that some JSON functions unsupported by TiFlash are pushed down to TiFlash [#9444](https://github.com/pingcap/tiflash/issues/9444) @[windtalker](https://github.com/windtalker) + - Fix the issue that setting the SSL certificate configuration to an empty string in TiFlash incorrectly enables TLS and causes TiFlash to fail to start [#9235](https://github.com/pingcap/tiflash/issues/9235) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that a network partition (network disconnection) between TiFlash and any PD might cause read request timeout errors [#9243](https://github.com/pingcap/tiflash/issues/9243) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that a large number of duplicate rows might be read in FastScan mode after importing data via BR or TiDB Lightning [#9118](https://github.com/pingcap/tiflash/issues/9118) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that TiFlash fails to parse the table schema when the table contains Bit-type columns with a default value that contains invalid characters [#9461](https://github.com/pingcap/tiflash/issues/9461) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that queries with virtual generated columns might return incorrect results after late materialization is enabled [#9188](https://github.com/pingcap/tiflash/issues/9188) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that TiFlash might fail to synchronize schemas after executing `ALTER TABLE ... EXCHANGE PARTITION` across databases [#7296](https://github.com/pingcap/tiflash/issues/7296) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash might panic when a database is deleted shortly after creation [#9266](https://github.com/pingcap/tiflash/issues/9266) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that when using the `CAST()` function to convert a string to a datetime with a time zone or invalid characters, the result is incorrect [#8754](https://github.com/pingcap/tiflash/issues/8754) @[solotzg](https://github.com/solotzg) + - Fix the issue that TiFlash might return transiently incorrect results in high-concurrency read scenarios [#8845](https://github.com/pingcap/tiflash/issues/8845) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that the `SUBSTRING_INDEX()` function might cause TiFlash to crash in some corner cases [#9116](https://github.com/pingcap/tiflash/issues/9116) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that frequent `EXCHANGE PARTITION` and `DROP TABLE` operations over a long period in a cluster might slow down the replication of TiFlash table metadata and degrade the query performance [#9227](https://github.com/pingcap/tiflash/issues/9227) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that a query with an empty key range fails to correctly generate read tasks on TiFlash, which might block TiFlash queries [#9108](https://github.com/pingcap/tiflash/issues/9108) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that the sign in the result of the `CAST AS DECIMAL` function is incorrect in certain cases [#9301](https://github.com/pingcap/tiflash/issues/9301) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that the `SUBSTRING()` function does not support the `pos` and `len` arguments for certain integer types, causing query errors [#9473](https://github.com/pingcap/tiflash/issues/9473) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that executing `DROP TABLE` on large tables might cause TiFlash OOM [#9437](https://github.com/pingcap/tiflash/issues/9437) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that BR integration test cases are unstable, and add a new test case to simulate snapshot or log backup file corruption [#53835](https://github.com/pingcap/tidb/issues/53835) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that DDLs requiring backfilling, such as `ADD INDEX` and `MODIFY COLUMN`, might not be correctly recovered during incremental restore [#54426](https://github.com/pingcap/tidb/issues/54426) @[3pointer](https://github.com/3pointer) + - Fix the issue that after a log backup PITR task fails and you stop it, the safepoints related to that task are not properly cleared in PD [#17316](https://github.com/tikv/tikv/issues/17316) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that log backup might be paused after the advancer owner migration [#53561](https://github.com/pingcap/tidb/issues/53561) @[RidRisR](https://github.com/RidRisR) + - Fix the inefficiency issue in scanning DDL jobs during incremental backups [#54139](https://github.com/pingcap/tidb/issues/54139) @[3pointer](https://github.com/3pointer) + - Fix the issue that the backup performance during checkpoint backups is affected due to interruptions in seeking Region leaders [#17168](https://github.com/tikv/tikv/issues/17168) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that BR logs might print sensitive credential information when log backup is enabled [#55273](https://github.com/pingcap/tidb/issues/55273) @[RidRisR](https://github.com/RidRisR) + - Fix the issue that BR fails to correctly identify errors due to multiple nested retries during the restore process [#54053](https://github.com/pingcap/tidb/issues/54053) @[RidRisR](https://github.com/RidRisR) + - Fix the issue that TiKV might panic when resuming a paused log backup task with unstable network connections to PD [#17020](https://github.com/tikv/tikv/issues/17020) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that backup tasks might get stuck if TiKV becomes unresponsive during the backup process [#53480](https://github.com/pingcap/tidb/issues/53480) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the checkpoint path of backup and restore is incompatible with some external storage [#55265](https://github.com/pingcap/tidb/issues/55265) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the Region fetched from PD does not have a Leader when restoring data using BR or importing data using TiDB Lightning in physical import mode [#51124](https://github.com/pingcap/tidb/issues/51124) [#50501](https://github.com/pingcap/tidb/issues/50501) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the transfer of PD leaders might cause BR to panic when restoring data [#53724](https://github.com/pingcap/tidb/issues/53724) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that after pausing, stopping, and rebuilding the log backup task, the task status is normal, but the checkpoint does not advance [#53047](https://github.com/pingcap/tidb/issues/53047) @[RidRisR](https://github.com/RidRisR) + - Fix the issue that log backups cannot resolve residual locks promptly, causing the checkpoint to fail to advance [#57134](https://github.com/pingcap/tidb/issues/57134) @[3pointer](https://github.com/3pointer) + + + TiCDC + + - Fix the issue that the default value of `TIMEZONE` type is not set according to the correct time zone [#10931](https://github.com/pingcap/tiflow/issues/10931) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that TiCDC might panic when the Sorter module reads disk data [#10853](https://github.com/pingcap/tiflow/issues/10853) @[hicqu](https://github.com/hicqu) + - Fix the issue that data inconsistency might occur when restarting Changefeed repeatedly when performing a large number of `UPDATE` operations in a multi-node environment [#11219](https://github.com/pingcap/tiflow/issues/11219) @[lidezhu](https://github.com/lidezhu) + - Fix the issue that after filtering out `add table partition` events is configured in `ignore-event`, TiCDC does not replicate other types of DML changes for related partitions to the downstream [#10524](https://github.com/pingcap/tiflow/issues/10524) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that `DROP PRIMARY KEY` and `DROP UNIQUE KEY` statements are not replicated correctly [#10890](https://github.com/pingcap/tiflow/issues/10890) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that the Processor module might get stuck when the downstream Kafka is inaccessible [#11340](https://github.com/pingcap/tiflow/issues/11340) @[asddongmen](https://github.com/asddongmen) + + + TiDB Data Migration (DM) + + - Fix the issue that DM does not set the default database when processing the `ALTER DATABASE` statement, which causes a replication error [#11503](https://github.com/pingcap/tiflow/issues/11503) @[lance6716](https://github.com/lance6716) + - Fix the issue that multiple DM-master nodes might simultaneously become leaders, leading to data inconsistency [#11602](https://github.com/pingcap/tiflow/issues/11602) @[GMHDBJD](https://github.com/GMHDBJD) + - Fix the connection blocking issue by upgrading `go-mysql` [#11041](https://github.com/pingcap/tiflow/issues/11041) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that data replication is interrupted when the index length exceeds the default value of `max-index-length` [#11459](https://github.com/pingcap/tiflow/issues/11459) @[michaelmdeng](https://github.com/michaelmdeng) + - Fix the issue that DM returns an error when replicating the `ALTER TABLE ... DROP PARTITION` statement for LIST partitioned tables [#54760](https://github.com/pingcap/tidb/issues/54760) @[lance6716](https://github.com/lance6716) + + + TiDB Lightning + + - Fix the issue that TiDB Lightning fails to receive oversized messages sent from TiKV [#56114](https://github.com/pingcap/tidb/issues/56114) @[fishiu](https://github.com/fishiu) + - Fix the issue that TiKV data might be corrupted when importing data after disabling the import mode of TiDB Lightning [#15003](https://github.com/tikv/tikv/issues/15003) [#47694](https://github.com/pingcap/tidb/issues/47694) @[lance6716](https://github.com/lance6716) + - Fix the issue that transaction conflicts occur during data import using TiDB Lightning [#49826](https://github.com/pingcap/tidb/issues/49826) @[lance6716](https://github.com/lance6716) + - Fix the issue that TiDB Lightning might fail to import data when EBS BR is running [#49517](https://github.com/pingcap/tidb/issues/49517) @[mittalrishabh](https://github.com/mittalrishabh) + - Fix the issue that TiDB Lightning reports a `verify allocator base failed` error when two instances simultaneously start parallel import tasks and are assigned the same task ID [#55384](https://github.com/pingcap/tidb/issues/55384) @[ei-sugimoto](https://github.com/ei-sugimoto) + - Fix the issue that killing the PD Leader causes TiDB Lightning to report the `invalid store ID 0` error during data import [#50501](https://github.com/pingcap/tidb/issues/50501) @[Leavrth](https://github.com/Leavrth) + + + Dumpling + + - Fix the issue that Dumpling reports an error when exporting tables and views at the same time [#53682](https://github.com/pingcap/tidb/issues/53682) @[tangenta](https://github.com/tangenta) + + + TiDB Binlog + + - Fix the issue that deleting rows during the execution of `ADD COLUMN` might report an error `data and columnID count not match` when TiDB Binlog is enabled [#53133](https://github.com/pingcap/tidb/issues/53133) @[tangenta](https://github.com/tangenta) diff --git a/releases/release-7.2.0.md b/releases/release-7.2.0.md index 7dad79e5729a1..76dd43ebde4d1 100644 --- a/releases/release-7.2.0.md +++ b/releases/release-7.2.0.md @@ -9,7 +9,7 @@ Release date: June 29, 2023 TiDB version: 7.2.0 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.2/quick-start-with-tidb) | [Installation packages](https://www.pingcap.com/download/?version=v7.2.0#version-list) +Quick access: [Quick start](https://docs-archive.pingcap.com/tidb/v7.2/quick-start-with-tidb) 7.2.0 introduces the following key features and improvements: @@ -24,22 +24,22 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.2/quick-start-with- Scalability and Performance - Resource groups support managing runaway queries (experimental) + Resource groups support managing runaway queries (experimental) You can now manage query timeout with more granularity, allowing for different behaviors based on query classifications. Queries meeting your specified threshold can be deprioritized or terminated. - TiFlash supports the pipeline execution model (experimental) + TiFlash supports the pipeline execution model (experimental) TiFlash supports a pipeline execution model to optimize thread resource control. SQL - Support a new SQL statement, IMPORT INTO, for data import (experimental) + Support a new SQL statement, IMPORT INTO, for data import (experimental) To simplify the deployment and maintenance of TiDB Lightning, TiDB introduces a new SQL statement IMPORT INTO, which integrates physical import mode of TiDB Lightning, including remote import from Amazon S3 or Google Cloud Storage (GCS) directly into TiDB. DB Operations and Observability - DDL supports pause and resume operations (experimental) + DDL supports pause and resume operations (experimental) This new capability lets you temporarily suspend resource-intensive DDL operations, such as index creation, to conserve resources and minimize the impact on online traffic. You can seamlessly resume these operations when ready, without the need to cancel and restart. This feature enhances resource utilization, improves user experience, and streamlines schema changes. @@ -109,7 +109,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.2/quick-start-with- To generate more reasonable execution plans, the behavior of the TiDB optimizer evolves over product iterations. However, in some particular scenarios, the changes might lead to performance regression. TiDB v7.2.0 introduces Optimizer Fix Controls to let you control some of the fine-grained behaviors of the optimizer. This enables you to roll back or control some new changes. - Each controllable behavior is described by a GitHub issue corresponding to the fix number. All controllable behaviors are listed in [Optimizer Fix Controls](/optimizer-fix-controls.md). You can set a target value for one or more behaviors by setting the [`tidb_opt_fix_control`](/system-variables.md#tidb_opt_fix_control-new-in-v710) system variable to achieve behavior control. + Each controllable behavior is described by a GitHub issue corresponding to the fix number. All controllable behaviors are listed in [Optimizer Fix Controls](/optimizer-fix-controls.md). You can set a target value for one or more behaviors by setting the [`tidb_opt_fix_control`](/system-variables.md#tidb_opt_fix_control-new-in-v653-and-v710) system variable to achieve behavior control. The Optimizer Fix Controls mechanism helps you control the TiDB optimizer at a granular level. It provides a new means of fixing performance issues caused by the upgrade process and improves the stability of TiDB. @@ -119,7 +119,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.2/quick-start-with- Starting from v7.2.0, the lightweight statistics initialization feature becomes GA. Lightweight statistics initialization can significantly reduce the number of statistics that must be loaded during startup, thus improving the speed of loading statistics. This feature increases the stability of TiDB in complex runtime environments and reduces the impact on the overall service when TiDB nodes restart. - For newly created clusters of v7.2.0 or later versions, TiDB loads lightweight statistics by default during TiDB startup and will wait for the loading to finish before providing services. For clusters upgraded from earlier versions, you can set the TiDB configuration items [`lite-init-stats`](/tidb-configuration-file.md#lite-init-stats-new-in-v710) and [`force-init-stats`](/tidb-configuration-file.md#force-init-stats-new-in-v710) to `true` to enable this feature. + For newly created clusters of v7.2.0 or later versions, TiDB loads lightweight statistics by default during TiDB startup and will wait for the loading to finish before providing services. For clusters upgraded from earlier versions, you can set the TiDB configuration items [`lite-init-stats`](/tidb-configuration-file.md#lite-init-stats-new-in-v710) and [`force-init-stats`](/tidb-configuration-file.md#force-init-stats-new-in-v657-and-v710) to `true` to enable this feature. For more information, see [documentation](/statistics.md#load-statistics). @@ -146,7 +146,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.2/quick-start-with- ADMIN RESUME DDL JOBS 1,2; ``` - For more information, see [documentation](/ddl-introduction.md#ddl-related-commands). + For more information, see [documentation](/best-practices/ddl-introduction.md#ddl-related-commands). ### Data migration @@ -154,7 +154,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.2/quick-start-with- The `IMPORT INTO` statement integrates the [Physical Import Mode](/tidb-lightning/tidb-lightning-physical-import-mode.md) capability of TiDB Lightning. With this statement, you can quickly import data in formats such as CSV, SQL, and PARQUET into an empty table in TiDB. This import method eliminates the need for a separate deployment and management of TiDB Lightning, thereby reducing the complexity of data import and greatly improving import efficiency. - For data files stored in Amazon S3 or GCS, when the [Backend task distributed execution framework](/tidb-distributed-execution-framework.md) is enabled, `IMPORT INTO` also supports splitting a data import job into multiple sub-jobs and scheduling them to multiple TiDB nodes for parallel import, which further enhances import performance. + For data files stored in Amazon S3 or GCS, when the [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md) is enabled, `IMPORT INTO` also supports splitting a data import job into multiple sub-jobs and scheduling them to multiple TiDB nodes for parallel import, which further enhances import performance. For more information, see [documentation](/sql-statements/sql-statement-import-into.md). @@ -170,6 +170,11 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.2/quick-start-with- > > This section provides compatibility changes you need to know when you upgrade from v7.1.0 to the current version (v7.2.0). If you are upgrading from v7.0.0 or earlier versions to the current version, you might also need to check the compatibility changes introduced in intermediate versions. +### Behavior changes + +- When processing update event, TiCDC splits an event into delete and insert events if the primary key or non-null unique index value is modified in the event. For more information, see [documentation](/ticdc/ticdc-split-update-behavior.md#transactions-containing-a-single-update-change). +- Changes the default value of [`tidb_remove_orderby_in_subquery`](/system-variables.md#tidb_remove_orderby_in_subquery-new-in-v610) from `OFF` to `ON`, meaning that the optimizer removes `ORDER BY` clauses from subqueries to avoid unnecessary sorting operations. This change might result in a different order of rows in query results. The ISO/IEC SQL standard does not require query results to follow the `ORDER BY` sorting specified in subqueries. If you need to ensure a specific order in the final result, add an `ORDER BY` clause to the outer query. If your applications rely on subquery sorting, you can set this variable to `OFF`. Clusters upgraded from earlier versions retain the previous behavior by default. + ### System variables | Variable name | Change type | Description | @@ -180,7 +185,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.2/quick-start-with- | [`tidb_analyze_skip_column_types`](/system-variables.md#tidb_analyze_skip_column_types-new-in-v720) | Newly added | Controls which types of columns are skipped for statistics collection when executing the `ANALYZE` command to collect statistics. The variable is only applicable for [`tidb_analyze_version = 2`](/system-variables.md#tidb_analyze_version-new-in-v510). When using the syntax of `ANALYZE TABLE t COLUMNS c1, ..., cn`, if the type of a specified column is included in `tidb_analyze_skip_column_types`, the statistics of this column will not be collected. | | [`tidb_enable_check_constraint`](/system-variables.md#tidb_enable_check_constraint-new-in-v720) | Newly added | Controls whether to enable `CHECK` constraints. The default value is `OFF`, which means this feature is disabled. | | [`tidb_enable_fast_table_check`](/system-variables.md#tidb_enable_fast_table_check-new-in-v720) | Newly added | Controls whether to use a checksum-based approach to quickly check the consistency of data and indexes in a table. The default value is `ON`, which means this feature is enabled. | -| [`tidb_enable_tiflash_pipeline_model`](https://docs.pingcap.com/tidb/v7.2/system-variables#tidb_enable_tiflash_pipeline_model-new-in-v720) | Newly added | Controls whether to enable the new execution model of TiFlash, the [pipeline model](/tiflash/tiflash-pipeline-model.md). The default value is `OFF`, which means the pipeline model is disabled. | +| [`tidb_enable_tiflash_pipeline_model`](https://docs-archive.pingcap.com/tidb/v7.2/system-variables#tidb_enable_tiflash_pipeline_model-new-in-v720) | Newly added | Controls whether to enable the new execution model of TiFlash, the [pipeline model](/tiflash/tiflash-pipeline-model.md). The default value is `OFF`, which means the pipeline model is disabled. | | [`tidb_expensive_txn_time_threshold`](/system-variables.md#tidb_expensive_txn_time_threshold-new-in-v720) | Newly added | Controls the threshold for logging expensive transactions, which is 600 seconds by default. When the duration of a transaction exceeds the threshold, and the transaction is neither committed nor rolled back, it is considered an expensive transaction and will be logged. | ### Configuration file parameters @@ -188,7 +193,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.2/quick-start-with- | Configuration file | Configuration parameter | Change type | Description | | -------- | -------- | -------- | -------- | | TiDB | [`lite-init-stats`](/tidb-configuration-file.md#lite-init-stats-new-in-v710) | Modified | Changes the default value from `false` to `true` after further tests, meaning that TiDB uses lightweight statistics initialization by default during TiDB startup to improve the initialization efficiency. | -| TiDB | [`force-init-stats`](/tidb-configuration-file.md#force-init-stats-new-in-v710) | Modified | Changes the default value from `false` to `true` to align with [`lite-init-stats`](/tidb-configuration-file.md#lite-init-stats-new-in-v710), meaning that TiDB waits for statistics initialization to finish before providing services during TiDB startup. | +| TiDB | [`force-init-stats`](/tidb-configuration-file.md#force-init-stats-new-in-v657-and-v710) | Modified | Changes the default value from `false` to `true` to align with [`lite-init-stats`](/tidb-configuration-file.md#lite-init-stats-new-in-v710), meaning that TiDB waits for statistics initialization to finish before providing services during TiDB startup. | | TiKV | [rocksdb.\[defaultcf\|writecf\|lockcf\].compaction-guard-min-output-file-size](/tikv-configuration-file.md#compaction-guard-min-output-file-size) | Modified | Changes the default value from `"8MB"` to `"1MB"` to reduce the data volume of compaction tasks in RocksDB. | | TiKV | [rocksdb.\[defaultcf\|writecf\|lockcf\].optimize-filters-for-memory](/tikv-configuration-file.md#optimize-filters-for-memory-new-in-v720) | Newly added | Controls whether to generate Bloom/Ribbon filters that minimize memory internal fragmentation. | | TiKV | [rocksdb.\[defaultcf\|writecf\|lockcf\].periodic-compaction-seconds](/tikv-configuration-file.md#periodic-compaction-seconds-new-in-v720) | Newly added | Controls the time interval for periodic compaction. SST files with updates older than this value will be selected for compaction and rewritten to the same level where these SST files originally reside. | @@ -236,7 +241,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.2/quick-start-with- + TiCDC - Optimize the structure of the directory where data files are stored when a DDL operation occurs in the scenario of replication to an object storage service [#8891](https://github.com/pingcap/tiflow/issues/8891) @[CharlesCheung96](https://github.com/CharlesCheung96) - - Support the OAUTHBEARER authentication in the scenario of replication to Kafka [#8865](https://github.com/pingcap/tiflow/issues/8865) @[hi-rustin](https://github.com/hi-rustin) + - Support the OAUTHBEARER authentication in the scenario of replication to Kafka [#8865](https://github.com/pingcap/tiflow/issues/8865) @[hi-rustin](https://github.com/Rustin170506) - Add the option of outputting only the handle keys for the `DELETE` operation in the scenario of replication to Kafka [#9143](https://github.com/pingcap/tiflow/issues/9143) @[3AceShowHand](https://github.com/3AceShowHand) + TiDB Data Migration (DM) @@ -245,7 +250,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.2/quick-start-with- + TiDB Lightning - - Optimize the retry mechanism during import to avoid errors caused by leader switching [#44478](https://github.com/pingcap/tidb/pull/44478) @[lance6716](https://github.com/lance6716) + - Optimize the retry mechanism during import to avoid errors caused by leader switching [#44263](https://github.com/pingcap/tidb/issues/44263) @[lance6716](https://github.com/lance6716) - Verify checksum through SQL after the import to improve stability of verification [#41941](https://github.com/pingcap/tidb/issues/41941) @[GMHDBJD](https://github.com/GMHDBJD) - Optimize TiDB Lightning OOM issues when importing wide tables [#43853](https://github.com/pingcap/tidb/issues/43853) @[D3Hunter](https://github.com/D3Hunter) @@ -259,6 +264,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.2/quick-start-with- - Fix the issue that the stale read global optimization does not take effect due to the lack of `TxnScope` in Coprocessor tasks [#43365](https://github.com/pingcap/tidb/issues/43365) @[you06](https://github.com/you06) - Fix the issue that follower read does not handle flashback errors before retrying, which causes query errors [#43673](https://github.com/pingcap/tidb/issues/43673) @[you06](https://github.com/you06) - Fix the issue that data and indexes are inconsistent when the `ON UPDATE` statement does not correctly update the primary key [#44565](https://github.com/pingcap/tidb/issues/44565) @[zyguan](https://github.com/zyguan) + - Fix the case sensitivity issue in some columns of the permission table [#41048](https://github.com/pingcap/tidb/issues/41048) @[bb7133](https://github.com/bb7133) - Modify the upper limit of the `UNIX_TIMESTAMP()` function to `3001-01-19 03:14:07.999999 UTC` to be consistent with that of MySQL 8.0.28 or later versions [#43987](https://github.com/pingcap/tidb/issues/43987) @[YangKeao](https://github.com/YangKeao) - Fix the issue that adding an index fails in the ingest mode [#44137](https://github.com/pingcap/tidb/issues/44137) @[tangenta](https://github.com/tangenta) - Fix the issue that canceling a DDL task in the rollback state causes errors in related metadata [#44143](https://github.com/pingcap/tidb/issues/44143) @[wjhuang2016](https://github.com/wjhuang2016) @@ -300,10 +306,10 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.2/quick-start-with- - Fix the issue that Resolved TS does not advance properly in some cases [#8963](https://github.com/pingcap/tiflow/issues/8963) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the issue that the `UPDATE` operation cannot output old values when the Avro or CSV protocol is used [#9086](https://github.com/pingcap/tiflow/issues/9086) @[3AceShowHand](https://github.com/3AceShowHand) - - Fix the issue of excessive downstream pressure caused by reading downstream metadata too frequently when replicating data to Kafka [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue of excessive downstream pressure caused by reading downstream metadata too frequently when replicating data to Kafka [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue of too many downstream logs caused by frequently setting the downstream bidirectional replication-related variables when replicating data to TiDB or MySQL [#9180](https://github.com/pingcap/tiflow/issues/9180) @[asddongmen](https://github.com/asddongmen) - Fix the issue that the PD node crashing causes the TiCDC node to restart [#8868](https://github.com/pingcap/tiflow/issues/8868) @[asddongmen](https://github.com/asddongmen) - - Fix the issue that TiCDC cannot create a changefeed with a downstream Kafka-on-Pulsar [#8892](https://github.com/pingcap/tiflow/issues/8892) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that TiCDC cannot create a changefeed with a downstream Kafka-on-Pulsar [#8892](https://github.com/pingcap/tiflow/issues/8892) @[hi-rustin](https://github.com/Rustin170506) + TiDB Lightning @@ -320,7 +326,7 @@ We would like to thank the following contributors from the TiDB community: - [darraes](https://github.com/darraes) - [demoManito](https://github.com/demoManito) - [dhysum](https://github.com/dhysum) -- [HappyUncle](https://github.com/HappyUncle) +- [happy-v587](https://github.com/happy-v587) - [jiyfhust](https://github.com/jiyfhust) - [L-maple](https://github.com/L-maple) - [nyurik](https://github.com/nyurik) diff --git a/releases/release-7.3.0.md b/releases/release-7.3.0.md index dc3ab1152cc16..fa8b53a66cfb9 100644 --- a/releases/release-7.3.0.md +++ b/releases/release-7.3.0.md @@ -9,7 +9,7 @@ Release date: August 14, 2023 TiDB version: 7.3.0 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.3/quick-start-with-tidb) | [Installation packages](https://www.pingcap.com/download/?version=v7.3.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.3/quick-start-with-tidb) 7.3.0 introduces the following major features. In addition to that, 7.3.0 also includes a series of enhancements (described in the [Feature details](#feature-details) section) to query stability in TiDB server and TiFlash. These enhancements are more miscellaneous in nature and not user-facing so they are not included in the following table. @@ -155,6 +155,10 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.3/quick-start-with- ### Behavior changes +* TiDB + + - MPP is a distributed computing framework provided by the TiFlash engine, which allows data exchange between nodes and provides high-performance, high-throughput SQL algorithms. Compared with other protocols, the MPP protocol is more mature and can provide better task and resource management. Starting from v7.3.0, when TiDB pushes computation tasks to TiFlash, the optimizer only generates execution plans using the MPP protocol by default. If [`tidb_allow_mpp`](/system-variables.md#tidb_allow_mpp-new-in-v50) is set to `OFF`, queries might return errors after you upgrade TiDB. It is recommended that you check the value of `tidb_allow_mpp` and set it to `ON` before the upgrade. If you still need the optimizer to choose one of the Cop, BatchCop, and MPP protocols for generating execution plans based on cost estimates, you can set the [`tidb_allow_tiflash_cop`](/system-variables.md#tidb_allow_tiflash_cop-new-in-v730) variable to `ON`. + * Backup & Restore (BR) - BR adds an empty cluster check before performing a full data restoration. By default, restoring data to a non-empty cluster is not allowed. If you want to force the restoration, you can use the `--filter` option to specify the corresponding table name to restore data to. @@ -175,6 +179,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.3/quick-start-with- | Variable name | Change type | Description | |--------|------------------------------|------| | [`tidb_opt_enable_mpp_shared_cte_execution`](/system-variables.md#tidb_opt_enable_mpp_shared_cte_execution-new-in-v720) | Modified | This system variable takes effect starting from v7.3.0. It controls whether non-recursive Common Table Expressions (CTEs) can be executed in TiFlash MPP. | +| [`tidb_allow_tiflash_cop`](/system-variables.md#tidb_allow_tiflash_cop-new-in-v730) | Newly added | This system variable is used to select the protocol for generating execution plans when TiDB pushes computation tasks down to TiFlash. | | [`tidb_lock_unchanged_keys`](/system-variables.md#tidb_lock_unchanged_keys-new-in-v711-and-v730) | Newly added | This variable is used to control in certain scenarios whether to lock the keys that are involved but not modified in a transaction. | | [`tidb_opt_enable_non_eval_scalar_subquery`](/system-variables.md#tidb_opt_enable_non_eval_scalar_subquery-new-in-v730) | Newly added | Controls whether the `EXPLAIN` statement disables the execution of constant subqueries that can be expanded at the optimization stage. | | [`tidb_skip_missing_partition_stats`](/system-variables.md#tidb_skip_missing_partition_stats-new-in-v730) | Newly added | This variable controls the generation of GlobalStats when partition statistics are missing. | diff --git a/releases/release-7.4.0.md b/releases/release-7.4.0.md index 19faf3150fb49..af131dc40c5e3 100644 --- a/releases/release-7.4.0.md +++ b/releases/release-7.4.0.md @@ -9,7 +9,7 @@ Release date: October 12, 2023 TiDB version: 7.4.0 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.4/quick-start-with-tidb) | [Installation packages](https://www.pingcap.com/download/?version=v7.4.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.4/quick-start-with-tidb) 7.4.0 introduces the following key features and improvements: @@ -25,7 +25,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.4/quick-start-with- Reliability and Availability Improve the performance and stability of IMPORT INTO and ADD INDEX operations via global sort (experimental) - Before v7.4.0, tasks such as ADD INDEX or IMPORT INTO using the distributed execution framework meant localized and partial sorting, which ultimately led to TiKV doing a lot of extra work to make up for the partial sorting. These jobs also required TiDB nodes to allocate local disk space for sorting, before loading to TiKV.
    With the introduction of the Global Sorting feature in v7.4.0, data is temporarily stored in external shared storage (S3 in this version) for global sorting before being loaded into TiKV. This eliminates the need for TiKV to consume extra resources and significantly improves the performance and stability of operations like ADD INDEX and IMPORT INTO. + Before v7.4.0, tasks such as ADD INDEX or IMPORT INTO using the TiDB Distributed eXecution Framework (DXF) meant localized and partial sorting, which ultimately led to TiKV doing a lot of extra work to make up for the partial sorting. These jobs also required TiDB nodes to allocate local disk space for sorting, before loading to TiKV.
    With the introduction of the Global Sort feature in v7.4.0, data is temporarily stored in external shared storage (S3 in this version) for global sorting before being loaded into TiKV. This eliminates the need for TiKV to consume extra resources and significantly improves the performance and stability of operations like ADD INDEX and IMPORT INTO. Resource control for background tasks (experimental) @@ -68,9 +68,9 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.4/quick-start-with- ### Scalability -* Support selecting the TiDB nodes to parallelly execute the backend `ADD INDEX` or `IMPORT INTO` tasks of the distributed execution framework (experimental) [#46453](https://github.com/pingcap/tidb/pull/46453) @[ywqzzy](https://github.com/ywqzzy) +* Support selecting the TiDB nodes to parallelly execute the backend `ADD INDEX` or `IMPORT INTO` tasks of the Distributed eXecution Framework (DXF) (experimental) [#46453](https://github.com/pingcap/tidb/pull/46453) @[ywqzzy](https://github.com/ywqzzy) - Executing `ADD INDEX` or `IMPORT INTO` tasks in parallel in a resource-intensive cluster can consume a large amount of TiDB node resources, which can lead to cluster performance degradation. Starting from v7.4.0, you can use the system variable [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) to control the service scope of each TiDB node under the [TiDB Backend Task Distributed Execution Framework](/tidb-distributed-execution-framework.md). You can select several existing TiDB nodes or set the TiDB service scope for new TiDB nodes, and all parallel `ADD INDEX` and `IMPORT INTO` tasks only run on these nodes. This mechanism can avoid performance impact on existing services. + Executing `ADD INDEX` or `IMPORT INTO` tasks in parallel in a resource-intensive cluster can consume a large amount of TiDB node resources, which can lead to cluster performance degradation. Starting from v7.4.0, you can use the system variable [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) to control the service scope of each TiDB node under the [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md). You can select several existing TiDB nodes or set the TiDB service scope for new TiDB nodes, and all parallel `ADD INDEX` and `IMPORT INTO` tasks only run on these nodes. This mechanism can avoid performance impact on existing services. For more information, see [documentation](/system-variables.md#tidb_service_scope-new-in-v740). @@ -106,7 +106,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.4/quick-start-with- * Introduce cloud storage-based global sort capability to improve the performance and stability of `ADD INDEX` and `IMPORT INTO` tasks in parallel execution (experimental) [#45719](https://github.com/pingcap/tidb/issues/45719) @[wjhuang2016](https://github.com/wjhuang2016) - Before v7.4.0, when executing tasks like `ADD INDEX` or `IMPORT INTO` in the distributed parallel execution framework, each TiDB node needs to allocate a significant amount of local disk space for sorting encoded index KV pairs and table data KV pairs. However, due to the lack of global sorting capability, there might be overlapping data between different TiDB nodes and within each individual node during the process. As a result, TiKV has to constantly perform compaction operations while importing these KV pairs into its storage engine, which impacts the performance and stability of `ADD INDEX` and `IMPORT INTO`. + Before v7.4.0, when executing tasks like `ADD INDEX` or `IMPORT INTO` in the Distributed eXecution Framework (DXF), each TiDB node needs to allocate a significant amount of local disk space for sorting encoded index KV pairs and table data KV pairs. However, due to the lack of global sorting capability, there might be overlapping data between different TiDB nodes and within each individual node during the process. As a result, TiKV has to constantly perform compaction operations while importing these KV pairs into its storage engine, which impacts the performance and stability of `ADD INDEX` and `IMPORT INTO`. In v7.4.0, TiDB introduces the [Global Sort](/tidb-global-sort.md) feature. Instead of writing the encoded data locally and sorting it there, the data is now written to cloud storage for global sorting. Once sorted, both the indexed data and table data are imported into TiKV in parallel, thereby improving performance and stability. @@ -185,9 +185,9 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.4/quick-start-with- For more information, see [documentation](/tidb-resource-control.md#manage-background-tasks). -* Enhance the ability to lock statistics [#46351](https://github.com/pingcap/tidb/issues/46351) @[hi-rustin](https://github.com/hi-rustin) +* Lock statistics becomes generally available (GA) [#46351](https://github.com/pingcap/tidb/issues/46351) @[hi-rustin](https://github.com/Rustin170506) - In v7.4.0, TiDB has enhanced the ability to [lock statistics](/statistics.md#lock-statistics). Now, to ensure operational security, locking and unlocking statistics require the same privileges as collecting statistics. In addition, TiDB supports locking and unlocking statistics for specific partitions, providing greater flexibility. If you are confident in queries and execution plans in the database and want to prevent any changes from occurring, you can lock statistics to enhance stability. + In v7.4.0, [lock statistics](/statistics.md#lock-statistics) becomes generally available. Now, to ensure operational security, locking and unlocking statistics require the same privileges as collecting statistics. In addition, TiDB supports locking and unlocking statistics for specific partitions, providing greater flexibility. If you are confident in queries and execution plans in the database and want to prevent any changes from occurring, you can lock statistics to enhance stability. For more information, see [documentation](/statistics.md#lock-statistics). @@ -195,9 +195,17 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.4/quick-start-with- MySQL 8.0 introduces hash joins for tables as a new feature. This feature is primarily used to join two relatively large tables and result sets. However, for transactional workloads, or some applications running on MySQL 5.7, hash joins for tables might pose a performance risk. MySQL provides the [`optimizer_switch`](https://dev.mysql.com/doc/refman/8.0/en/switchable-optimizations.html#optflag_block-nested-loop) to control whether to select hash joins at the global or session level. - Starting from v7.4.0, TiDB introduces the system variable [`tidb_opt_enable_hash_join`](/system-variables.md#tidb_opt_enable_hash_join-new-in-v740) to have control over hash joins for tables. It is enabled by default (`ON`). If you are sure that you do not need to select hash joins between tables in your execution plan, you can modify the variable to `OFF` to reduce the possibility of execution plan rollbacks and improve system stability. + Starting from v7.4.0, TiDB introduces the system variable [`tidb_opt_enable_hash_join`](/system-variables.md#tidb_opt_enable_hash_join-new-in-v656-v712-and-v740) to have control over hash joins for tables. It is enabled by default (`ON`). If you are sure that you do not need to select hash joins between tables in your execution plan, you can modify the variable to `OFF` to reduce the possibility of execution plan rollbacks and improve system stability. - For more information, see [documentation](/system-variables.md#tidb_opt_enable_hash_join-new-in-v740). + For more information, see [documentation](/system-variables.md#tidb_opt_enable_hash_join-new-in-v656-v712-and-v740). + +* Memory control for the statistics cache is generally available (GA) [#45367](https://github.com/pingcap/tidb/issues/45367) @[hawkingrei](https://github.com/hawkingrei) + + TiDB instances can cache table statistics to accelerate execution plan generation and improve SQL performance. Starting from v6.1.0, TiDB introduces the system variable [`tidb_stats_cache_mem_quota`](https://github.com/system-variables.md#tidb_stats_cache_mem_quota-new-in-v610). By configuring this system variable, you can set a memory usage limit for the statistics cache. When the cache reaches its limit, TiDB automatically evicts inactive cache entries, helping control instance memory usage and improve stability. + + Starting from v7.4.0, this feature becomes generally available (GA). + + For more information, see [documentation](/system-variables.md#tidb_stats_cache_mem_quota-new-in-v610). ### SQL @@ -241,7 +249,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.4/quick-start-with- * Enhance the `IMPORT INTO` feature [#46704](https://github.com/pingcap/tidb/issues/46704) @[D3Hunter](https://github.com/D3Hunter) - Starting from v7.4.0, you can add the `CLOUD_STORAGE_URI` option in the `IMPORT INTO` statement to enable the [global sorting](/tidb-global-sort.md) feature (experimental), which helps boost import performance and stability. In the `CLOUD_STORAGE_URI` option, you can specify a cloud storage address for the encoded data. + Starting from v7.4.0, you can add the `CLOUD_STORAGE_URI` option in the `IMPORT INTO` statement to enable the [Global Sort](/tidb-global-sort.md) feature (experimental), which helps boost import performance and stability. In the `CLOUD_STORAGE_URI` option, you can specify a cloud storage address for the encoded data. In addition, in v7.4.0, the `IMPORT INTO` feature introduces the following functionalities: @@ -286,6 +294,8 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.4/quick-start-with- - The [`information_schema.CHECK_CONSTRAINTS`](/information-schema/information-schema-check-constraints.md) table is added for improved compatibility with MySQL 8.0. +- For transactions containing multiple changes, if the primary key or non-null unique index value is modified in the update event, TiCDC splits an event into delete and insert events and ensures that all events follow the sequence of delete events preceding insert events. For more information, see [documentation](/ticdc/ticdc-split-update-behavior.md#transactions-containing-multiple-update-changes). + ### System variables | Variable name | Change type | Description | @@ -294,10 +304,11 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.4/quick-start-with- | [`tidb_enable_non_prepared_plan_cache`](/system-variables.md#tidb_enable_non_prepared_plan_cache) | Modified | Changes the default value from `ON` to `OFF` after further tests, meaning that non-prepared execution plan cache is disabled. | | [`default_collation_for_utf8mb4`](/system-variables.md#default_collation_for_utf8mb4-new-in-v740) | Newly added | Controls the default collation for the `utf8mb4` character set. The default value is `utf8mb4_bin`. | | [`tidb_cloud_storage_uri`](/system-variables.md#tidb_cloud_storage_uri-new-in-v740) | Newly added | Specifies the cloud storage URI to enable [Global Sort](/tidb-global-sort.md). | -| [`tidb_opt_enable_hash_join`](/system-variables.md#tidb_opt_enable_hash_join-new-in-v740) | Newly added | Controls whether the optimizer will select hash joins for tables. The value is `ON` by default. If set to `OFF`, the optimizer avoids selecting a hash join of a table unless there is no other execution plan available. | +| [`tidb_opt_enable_hash_join`](/system-variables.md#tidb_opt_enable_hash_join-new-in-v656-v712-and-v740) | Newly added | Controls whether the optimizer will select hash joins for tables. The value is `ON` by default. If set to `OFF`, the optimizer avoids selecting a hash join of a table unless there is no other execution plan available. | | [`tidb_opt_objective`](/system-variables.md#tidb_opt_objective-new-in-v740) | Newly added | This variable controls the objective of the optimizer. `moderate` maintains the default behavior in versions prior to TiDB v7.4.0, where the optimizer tries to use more information to generate better execution plans. `determinate` mode tends to be more conservative and makes the execution plan more stable. | +| [`tidb_request_source_type`](/system-variables.md#tidb_request_source_type-new-in-v740) | Newly added | Explicitly specifies the task type for the current session, which is identified and controlled by [Resource Control](/tidb-resource-control.md). For example: `SET @@tidb_request_source_type = "background"`. | | [`tidb_schema_version_cache_limit`](/system-variables.md#tidb_schema_version_cache_limit-new-in-v740) | Newly added | This variable limits how many historical schema versions can be cached in a TiDB instance. The default value is `16`, which means that TiDB caches 16 historical schema versions by default. | -| [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) | Newly added | This variable is an instance-level system variable. You can use it to control the service scope of TiDB nodes under the [TiDB distributed execution framework](/tidb-distributed-execution-framework.md). When you set `tidb_service_scope` of a TiDB node to `background`, the TiDB distributed execution framework schedules that TiDB node to execute background tasks, such as [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) and [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md). | +| [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) | Newly added | This variable is an instance-level system variable. You can use it to control the service scope of TiDB nodes under the [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md). When you set `tidb_service_scope` of a TiDB node to `background`, the DXF schedules that TiDB node to execute DXF tasks, such as [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) and [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md). | | [`tidb_session_alias`](/system-variables.md#tidb_session_alias-new-in-v740) | Newly added | Controls the value of the `session_alias` column in the logs related to the current session. | | [`tiflash_mem_quota_query_per_node`](/system-variables.md#tiflash_mem_quota_query_per_node-new-in-v740) | Newly added | Limits the maximum memory usage for a query on a TiFlash node. When the memory usage of a query exceeds this limit, TiFlash returns an error and terminates the query. The default value is `0`, which means no limit.| | [`tiflash_query_spill_ratio`](/system-variables.md#tiflash_query_spill_ratio-new-in-v740) | Newly added | Controls the threshold for TiFlash [query-level spilling](/tiflash/tiflash-spill-disk.md#query-level-spilling). The default value is `0.7`. | @@ -318,10 +329,11 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.4/quick-start-with- | TiCDC | [`large-message-handle-compression`](/ticdc/ticdc-sink-to-kafka.md#ticdc-data-compression) | Newly added | Controls whether to enable compression during encoding. The default value is empty, which means not enabled. | | TiCDC | [`large-message-handle-option`](/ticdc/ticdc-sink-to-kafka.md#send-large-messages-to-external-storage) | Modified | This configuration item adds a new value `claim-check`. When it is set to `claim-check`, TiCDC Kafka sink supports sending the message to external storage when the message size exceeds the limit and sends a message to Kafka containing the address of this large message in external storage. | -## Deprecated features +## Deprecated and removed features + [Mydumper](https://docs.pingcap.com/tidb/v4.0/mydumper-overview) will be deprecated in v7.5.0 and most of its features have been replaced by [Dumpling](/dumpling-overview.md). It is strongly recommended that you use Dumpling instead of mydumper. + TiKV-importer will be deprecated in v7.5.0. It is strongly recommended that you use the [Physical Import Mode of TiDB Lightning](/tidb-lightning/tidb-lightning-physical-import-mode.md) as an alternative. ++ The `enable-old-value` parameter of TiCDC is removed. [#9667](https://github.com/pingcap/tiflow/issues/9667) @[3AceShowHand](https://github.com/3AceShowHand) ## Improvements @@ -446,7 +458,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.4/quick-start-with- - Fix an issue that the misleading error message `resolve lock timeout` covers up the actual error when backup fails [#43236](https://github.com/pingcap/tidb/issues/43236) @[YuJuncen](https://github.com/YuJuncen) - Fix the issue that recovering implicit primary keys using PITR might cause conflicts [#46520](https://github.com/pingcap/tidb/issues/46520) @[3pointer](https://github.com/3pointer) - Fix the issue that recovering meta-kv using PITR might cause errors [#46578](https://github.com/pingcap/tidb/issues/46578) @[Leavrth](https://github.com/Leavrth) - - Fix the errors in BR integration test cases [#45561](https://github.com/pingcap/tidb/issues/46561) @[purelind](https://github.com/purelind) + - Fix the errors in BR integration test cases [#46561](https://github.com/pingcap/tidb/issues/46561) @[purelind](https://github.com/purelind) + TiCDC @@ -493,4 +505,4 @@ We would like to thank the following contributors from the TiDB community: - [shawn0915](https://github.com/shawn0915) - [tedyu](https://github.com/tedyu) - [yumchina](https://github.com/yumchina) -- [ZhuohaoHe](https://github.com/ZhuohaoHe) +- [ZzzhHe](https://github.com/ZzzhHe) diff --git a/releases/release-7.5.0.md b/releases/release-7.5.0.md new file mode 100644 index 0000000000000..cb6b6c6c9db44 --- /dev/null +++ b/releases/release-7.5.0.md @@ -0,0 +1,307 @@ +--- +title: TiDB 7.5.0 Release Notes +summary: Learn about the new features, compatibility changes, improvements, and bug fixes in TiDB 7.5.0. +--- + +# TiDB 7.5.0 Release Notes + + + +Release date: December 1, 2023 + +TiDB version: 7.5.0 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.5/production-deployment-using-tiup) + +TiDB 7.5.0 is a Long-Term Support Release (LTS). + +Compared with the previous LTS 7.1.0, 7.5.0 includes new features, improvements, and bug fixes released in [7.2.0-DMR](/releases/release-7.2.0.md), [7.3.0-DMR](/releases/release-7.3.0.md), and [7.4.0-DMR](/releases/release-7.4.0.md). When you upgrade from 7.1.x to 7.5.0, you can download the [TiDB Release Notes PDF](https://docs-download.pingcap.com/pdf/tidb-v7.2-to-v7.5-en-release-notes.pdf) to view all release notes between the two LTS versions. The following table lists some highlights from 7.2.0 to 7.5.0: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    CategoryFeatureDescription
    Scalability and PerformanceSupport running multiple ADD INDEX statements in parallelThis feature allows for concurrent jobs to add multiple indexes for a single table. Previously, it would take the time of X plus the time of Y to execute two ADD INDEX statements simultaneously (X and Y). With this feature, adding two indexes X and Y in one SQL can be concurrently executed, and the total execution time of DDL is significantly reduced. Especially in scenarios with wide tables, internal test data shows that performance can be improved by up to 94%.
    Reliability and AvailabilityOptimize Global Sort (experimental, introduced in v7.4.0)TiDB v7.1.0 introduced the Distributed eXecution Framework (DXF). For tasks that take advantage of this framework, v7.4 introduces global sorting to eliminate the unnecessary I/O, CPU, and memory spikes caused by temporarily out-of-order data during data re-organization tasks. The global sorting takes advantage of external shared object storage (Amazon S3 in this first iteration) to store intermediary files during the job, adding flexibility and cost savings. Operations like ADD INDEX and IMPORT INTO will be faster, more resilient, more stable, more flexible, and cost less to run.
    Resource control for background tasks (experimental, introduced in v7.4.0)In v7.1.0, the Resource Control feature was introduced to mitigate resource and storage access interference between workloads. TiDB v7.4.0 applied this control to the priority of background tasks as well. In v7.4.0, Resource Control now identifies and manages the priority of background task execution, such as auto-analyze, Backup & Restore, bulk load with TiDB Lightning, and online DDL. In future releases, this control will eventually apply to all background tasks.
    Resource control for managing runaway queries (experimental, introduced in v7.2.0)Resource Control is a framework for resource-isolating workloads by Resource Groups, but it makes no calls on how individual queries affect work inside of each group. TiDB v7.2.0 introduces "runaway queries control" to let you control how TiDB identifies and treats these queries per Resource Group. Depending on needs, long running queries might be terminated or throttled, and the queries can be identified by exact SQL text, SQL digests or their plan digests, for better generalization. In v7.3.0, TiDB enables you to proactively watch for known bad queries, similar to a SQL blocklist at the database level.
    SQLMySQL 8.0 compatibility (introduced in v7.4.0)In MySQL 8.0, the default character set is utf8mb4, and the default collation of utf8mb4 is utf8mb4_0900_ai_ci. TiDB v7.4.0 adding support for this enhances compatibility with MySQL 8.0 so that migrations and replications from MySQL 8.0 databases with the default collation are now much smoother.
    DB Operations and ObservabilityTiDB Lightning's physical import mode integrated into TiDB with IMPORT INTO (GA)Before v7.2.0, to import data based on the file system, you needed to install TiDB Lightning and used its physical import mode. Now, the same capability is integrated into the IMPORT INTO statement so you can use this statement to quickly import data without installing any additional tool. This statement also supports the Distributed eXecution Framework (DXF) for parallel import, which improves import efficiency during large-scale imports.
    Specify the respective TiDB nodes to execute the ADD INDEX and IMPORT INTO SQL statements (GA)You have the flexibility to specify whether to execute ADD INDEX or IMPORT INTO SQL statements on some of the existing TiDB nodes or newly added TiDB nodes. This approach enables resource isolation from the rest of the TiDB nodes, preventing any impact on business operations while ensuring optimal performance for executing the preceding SQL statements. In v7.5.0, this feature becomes generally available (GA).
    DDL supports pause and resume operations (GA)Adding indexes can be big resource consumers and can affect online traffic. Even when throttled in a Resource Group or isolated to labeled nodes, there may still be a need to suspend these jobs in emergencies. As of v7.2.0, TiDB now natively supports suspending any number of these background jobs at once, freeing up needed resources while avoiding having to cancel and restart the jobs.
    TiDB Dashboard supports heap profiling for TiKVPreviously, addressing TiKV OOM or high memory usage issues typically required manual execution of jeprof to generate a heap profile in the instance environment. Starting from v7.5.0, TiKV enables remote processing of heap profiles. You can now directly access the flame graph and call graph of heap profile. This feature provides the same simple and easy-to-use experience as Go heap profiling.
    + +## Feature details + +### Scalability + +* Support designating and isolating TiDB nodes to distributedly execute `ADD INDEX` or `IMPORT INTO` tasks when the Distributed eXecution Framework (DXF) is enabled [#46258](https://github.com/pingcap/tidb/issues/46258) @[ywqzzy](https://github.com/ywqzzy) + + Executing `ADD INDEX` or `IMPORT INTO` tasks in parallel in a resource-intensive cluster can consume a large amount of TiDB node resources, which can lead to cluster performance degradation. To avoid performance impact on existing services, v7.4.0 introduces the system variable [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) as an experimental feature to control the service scope of each TiDB node under the [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md). You can select several existing TiDB nodes or set the TiDB service scope for new TiDB nodes, and all distributedly executed `ADD INDEX` and `IMPORT INTO` tasks only run on these nodes. In v7.5.0, this feature becomes generally available (GA). + + For more information, see [documentation](/system-variables.md#tidb_service_scope-new-in-v740). + +### Performance + +* The TiDB Distributed eXecution Framework (DXF) becomes generally available (GA), improving the performance and stability of `ADD INDEX` and `IMPORT INTO` tasks in parallel execution [#45719](https://github.com/pingcap/tidb/issues/45719) @[wjhuang2016](https://github.com/wjhuang2016) + + The DXF introduced in v7.1.0 has become GA. In versions before TiDB v7.1.0, only one TiDB node can execute DDL tasks at the same time. Starting from v7.1.0, multiple TiDB nodes can execute the same DDL task in parallel under the DXF. Starting from v7.2.0, the DXF supports multiple TiDB nodes to execute the same `IMPORT INTO` task in parallel, thereby better utilizing the resources of the TiDB cluster and significantly improving the performance of DDL and `IMPORT INTO` tasks. In addition, you can also increase TiDB nodes to linearly improve the performance of these tasks. + + To use the DXF, set [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) value to `ON`. + + ```sql + SET GLOBAL tidb_enable_dist_task = ON; + ``` + + For more information, see [documentation](/tidb-global-sort.md). + +* Improve the performance of adding multiple indexes in a single SQL statement [#41602](https://github.com/pingcap/tidb/issues/41602) @[tangenta](https://github.com/tangenta) + + Before v7.5.0, when you add multiple indexes (`ADD INDEX`) in a single SQL statement, the performance was similar to adding multiple indexes using separate SQL statements. Starting from v7.5.0, the performance of adding multiple indexes in a single SQL statement is significantly improved. Especially in scenarios with wide tables, internal test data shows that performance can be improved by up to 94%. + +### DB operations + +* DDL jobs support pause and resume operations (GA) [#18015](https://github.com/pingcap/tidb/issues/18015) @[godouxm](https://github.com/godouxm) + + The pause and resume operations for DDL jobs introduced in v7.2.0 become generally available (GA). These operations let you pause resource-intensive DDL jobs (such as creating indexes) to save resources and minimize the impact on online traffic. When resources permit, you can seamlessly resume DDL jobs without canceling and restarting them. This feature improves resource utilization, enhances user experience, and simplifies the schema change process. + + You can pause and resume multiple DDL jobs using `ADMIN PAUSE DDL JOBS` or `ADMIN RESUME DDL JOBS`: + + ```sql + ADMIN PAUSE DDL JOBS 1,2; + ADMIN RESUME DDL JOBS 1,2; + ``` + + For more information, see [documentation](/best-practices/ddl-introduction.md#ddl-related-commands). + +* BR supports backing up and restoring statistics [#48008](https://github.com/pingcap/tidb/issues/48008) @[Leavrth](https://github.com/Leavrth) + + Starting from TiDB v7.5.0, the br command-line tool introduces the `--ignore-stats` parameter to back up and restore database statistics. When you set this parameter to `false`, the br command-line tool supports backing up and restoring statistics of columns, indexes, and tables. In this case, you do not need to manually run the statistics collection task for the TiDB database restored from the backup, or wait for the completion of automatic collection tasks. This feature simplifies database maintenance work and improves query performance. + + For more information, see [documentation](/br/br-snapshot-manual.md#back-up-statistics). + +### Observability + +* TiDB Dashboard supports heap profiling for TiKV [#15927](https://github.com/tikv/tikv/issues/15927) @[Connor1996](https://github.com/Connor1996) + + Previously, addressing TiKV OOM or high memory usage issues typically required manual execution of `jeprof` to generate a heap profile in the instance environment. Starting from v7.5.0, TiKV enables remote processing of heap profiles. You can now directly access the flame graph and call graph of heap profile. This feature provides the same simple and easy-to-use experience as Go heap profiling. + + For more information, see [documentation](/dashboard/dashboard-profiling.md). + +### Data migration + +* Support the `IMPORT INTO` SQL statement (GA) [#46704](https://github.com/pingcap/tidb/issues/46704) @[D3Hunter](https://github.com/D3Hunter) + + In v7.5.0, the `IMPORT INTO` SQL statement becomes generally available (GA). This statement integrates the [Physical Import Mode](/tidb-lightning/tidb-lightning-physical-import-mode.md) capability of TiDB Lightning and allows you to quickly import data in formats such as CSV, SQL, and PARQUET into an empty table in TiDB. This import method eliminates the need for a separate deployment and management of TiDB Lightning, thereby reducing the complexity of data import and greatly improving import efficiency. + + For more information, see [documentation](/sql-statements/sql-statement-import-into.md). + +* Data Migration (DM) supports blocking incompatible (data-consistency-corrupting) DDL changes [#9692](https://github.com/pingcap/tiflow/issues/9692) @[GMHDBJD](https://github.com/GMHDBJD) + + Before v7.5.0, the DM Binlog Filter feature can only migrate or filter specified events, and the granularity is relatively coarse. For example, it can only filter large granularity of DDL events such as `ALTER`. This method is limited in some scenarios. For example, the application allows `ADD COLUMN` but not `DROP COLUMN`, but they are both filtered by `ALTER` events in the earlier DM versions. + + To address such issues, v7.5.0 refines the granularity of the supported DDL events, such as support filtering `MODIFY COLUMN` (modify the column data type), `DROP COLUMN`, and other fine-grained DDL events that lead to data loss, truncation of data, and loss of precision. You can configure it as needed. This feature also supports blocking incompatible DDL changes and reporting errors for such changes, so that you can intervene manually in time to avoid impacting downstream application data. + + For more information, see [documentation](/dm/dm-binlog-event-filter.md#parameter-descriptions). + +* Support real-time checkpoint updates for continuous data validation [#8463](https://github.com/pingcap/tiflow/issues/8463) @[lichunzhu](https://github.com/lichunzhu) + + Before v7.5.0, the [continuous data validation feature](/dm/dm-continuous-data-validation.md) ensures the data consistency during replication from DM to downstream. This serves as the basis for cutting over business traffic from the upstream database to TiDB. However, due to various factors such as replication delay and waiting for re-validation of inconsistent data, the continuous validation checkpoint must be refreshed every few minutes. This is unacceptable for some business scenarios where the cutover time is limited to tens of seconds. + + With the introduction of real-time updating of checkpoint for continuous data validation, you can now provide the binlog position from the upstream database. Once the continuous validation program detects this binlog position in memory, it immediately refreshes the checkpoint instead of refreshing it every few minutes. Therefore, you can quickly perform cut-off operations based on this immediately updated checkpoint. + + For more information, see [documentation](/dm/dm-continuous-data-validation.md#set-the-cutover-point-for-continuous-data-validation). + +## Compatibility changes + +> **Note:** +> +> This section provides compatibility changes you need to know when you upgrade from v7.4.0 to the current version (v7.5.0). If you are upgrading from v7.3.0 or earlier versions to the current version, you might also need to check the compatibility changes introduced in intermediate versions. + +### System variables + +| Variable name | Change type | Description | +|--------|------------------------------|------| +| [`tidb_enable_fast_analyze`](/system-variables.md#tidb_enable_fast_analyze) | Deprecated | Controls whether to enable the statistics `Fast Analyze` feature. This feature is deprecated in v7.5.0. | +| [`tidb_analyze_partition_concurrency`](/system-variables.md#tidb_analyze_partition_concurrency) | Modified | Changes the default value from `1` to `2` after further tests. | +| [`tidb_build_stats_concurrency`](/system-variables.md#tidb_build_stats_concurrency) | Modified | Changes the default value from `4` to `2` after further tests. | +| [`tidb_merge_partition_stats_concurrency`](/system-variables.md#tidb_merge_partition_stats_concurrency) | Modified | This system variable takes effect starting from v7.5.0. It specifies the concurrency of merging statistics for a partitioned table when TiDB analyzes the partitioned table. | +| [`tidb_build_sampling_stats_concurrency`](/system-variables.md#tidb_build_sampling_stats_concurrency-new-in-v750) | Newly added | Controls the sampling concurrency of the `ANALYZE` process. | +| [`tidb_enable_async_merge_global_stats`](/system-variables.md#tidb_enable_async_merge_global_stats-new-in-v750) | Newly added | This variable is used by TiDB to merge statistics asynchronously to avoid OOM issues. | +| [`tidb_gogc_tuner_max_value`](/system-variables.md#tidb_gogc_tuner_max_value-new-in-v750) | Newly added | Controls the maximum value of GOGC that the GOGC Tuner can adjust. | +| [`tidb_gogc_tuner_min_value`](/system-variables.md#tidb_gogc_tuner_min_value-new-in-v750) | Newly added | Controls the minimum value of GOGC that the GOGC Tuner can adjust.| + +### Configuration file parameters + +| Configuration file | Configuration parameter | Change type | Description | +| -------- | -------- | -------- | -------- | +| TiDB | [`tikv-client.copr-req-timeout`](/tidb-configuration-file.md#copr-req-timeout-new-in-v750) | Newly added | Sets the timeout of a single Coprocessor request. | +| TiKV | [`raftstore.inspect-interval`](/tikv-configuration-file.md#inspect-interval) | Modified | Changes the default value from `500ms` to `100ms` after optimizing the algorithm to improve the sensitivity of slow node detection. | +| TiKV | [`raftstore.region-compact-min-redundant-rows`](/tikv-configuration-file.md#region-compact-min-redundant-rows-new-in-v710) | Modified | Sets the number of redundant MVCC rows required to trigger RocksDB compaction. Starting from v7.5.0, this configuration item takes effect for the `"raft-kv"` storage engine. | +| TiKV | [`raftstore.region-compact-redundant-rows-percent`](/tikv-configuration-file.md#region-compact-redundant-rows-percent-new-in-v710) | Modified | Sets the percentage of redundant MVCC rows required to trigger RocksDB compaction. Starting from v7.5.0, this configuration item takes effect for the `"raft-kv"` storage engine. | +| TiKV | [`raftstore.evict-cache-on-memory-ratio`](/tikv-configuration-file.md#evict-cache-on-memory-ratio-new-in-v750) | Newly added | When the memory usage of TiKV exceeds 90% of the system available memory, and the memory occupied by Raft entry cache exceeds the `evict-cache-on-memory-ratio` of used memory, TiKV evicts the Raft entry cache. | +| TiKV | [`memory.enable-heap-profiling`](/tikv-configuration-file.md#enable-heap-profiling-new-in-v750) | Newly added | Controls whether to enable Heap Profiling to track the memory usage of TiKV. | +| TiKV | [`memory.profiling-sample-per-bytes`](/tikv-configuration-file.md#profiling-sample-per-bytes-new-in-v750) | Newly added | Specifies the amount of data sampled by Heap Profiling each time, rounding up to the nearest power of 2. | +| BR | [`--ignore-stats`](/br/br-snapshot-manual.md#back-up-statistics) | Newly added | Controls whether to back up and restore database statistics. When you set this parameter to `false`, the br command-line tool supports backing up and restoring statistics of columns, indexes, and tables. | +| TiCDC | [`case-sensitive`](/ticdc/ticdc-changefeed-config.md) | Modified | Changes the default value from `true` to `false` after further tests, which means that the table names and database names in the TiCDC configuration file are case-insensitive by default. | +| TiCDC | [`sink.dispatchers.partition`](/ticdc/ticdc-changefeed-config.md) | Modified | Controls how TiCDC dispatches incremental data to Kafka partitions. v7.5.0 introduces a new value option `columns`, which uses the explicitly specified column values to calculate the partition number. | +| TiCDC | [`changefeed-error-stuck-duration`](/ticdc/ticdc-changefeed-config.md) | Newly added | Controls the duration for which the changefeed is allowed to automatically retry when internal errors or exceptions occur. | +| TiCDC | [`encoding-worker-num`](/ticdc/ticdc-changefeed-config.md) | Newly added | Controls the number of encoding and decoding workers in the redo module. | +| TiCDC | [`flush-worker-num`](/ticdc/ticdc-changefeed-config.md) | Newly added | Controls the number of flushing workers in the redo module. | +| TiCDC | [`sink.column-selectors`](/ticdc/ticdc-changefeed-config.md) | Newly added | Controls the specified columns of data change events that TiCDC sends to Kafka when dispatching incremental data. | +| TiCDC | [`sql-mode`](/ticdc/ticdc-changefeed-config.md) | Newly added | Specifies the SQL mode used by TiCDC when parsing DDL statements. The default value is the same as the default SQL mode of TiDB. | +| TiDB Lightning | `--importer` | Deleted | Specifies the address of TiKV-importer, which is deprecated in v7.5.0. | + +## Offline package changes + +Starting from v7.5.0, the following contents are removed from the `TiDB-community-toolkit` [binary package](/binary-package.md): + +- `tikv-importer-{version}-linux-{arch}.tar.gz` +- `mydumper` +- `spark-{version}-any-any.tar.gz` +- `tispark-{version}-any-any.tar.gz` + +## Deprecated features + +* [Mydumper](https://docs.pingcap.com/tidb/v4.0/mydumper-overview) is deprecated in v7.5.0 and most of its features have been replaced by [Dumpling](/dumpling-overview.md). It is strongly recommended that you use Dumpling instead of Mydumper. + +* TiKV-importer is deprecated in v7.5.0. It is strongly recommended that you use the [Physical Import Mode of TiDB Lightning](/tidb-lightning/tidb-lightning-physical-import-mode.md) as an alternative. + +* Starting from TiDB v7.5.0, technical support for the data replication feature of [TiDB Binlog](/tidb-binlog/tidb-binlog-overview.md) is no longer provided. It is strongly recommended to use [TiCDC](/ticdc/ticdc-overview.md) as an alternative solution for data replication. Although TiDB Binlog v7.5.0 still supports the Point-in-Time Recovery (PITR) scenario, this component will be completely deprecated in future versions. It is recommended to use [PITR](/br/br-pitr-guide.md) as an alternative solution for data recovery. + +* The [`Fast Analyze`](/system-variables.md#tidb_enable_fast_analyze) feature (experimental) for statistics is deprecated in v7.5.0. + +* The [incremental collection](https://docs.pingcap.com/tidb/v7.4/statistics#incremental-collection) feature (experimental) for statistics is deprecated in v7.5.0. + +## Improvements + ++ TiDB + + - Optimize the concurrency model of merging GlobalStats: introduce [`tidb_enable_async_merge_global_stats`](/system-variables.md#tidb_enable_async_merge_global_stats-new-in-v750) to enable simultaneous loading and merging of statistics, which speeds up the generation of GlobalStats on partitioned tables. Optimize the memory usage of merging GlobalStats to avoid OOM and reduce memory allocations. [#47219](https://github.com/pingcap/tidb/issues/47219) @[hawkingrei](https://github.com/hawkingrei) + - Optimize the `ANALYZE` process: introduce [`tidb_build_sampling_stats_concurrency`](/system-variables.md#tidb_build_sampling_stats_concurrency-new-in-v750) to better control the `ANALYZE` concurrency to reduce resource consumption. Optimize the memory usage of `ANALYZE` to reduce memory allocation and avoid frequent GC by reusing some intermediate results. [#47275](https://github.com/pingcap/tidb/issues/47275) @[hawkingrei](https://github.com/hawkingrei) + - Optimize the use of placement policies: support configuring the range of a policy to global and improve the syntax support for common scenarios. [#45384](https://github.com/pingcap/tidb/issues/45384) @[nolouch](https://github.com/nolouch) + - Improve the performance of adding indexes with `tidb_ddl_enable_fast_reorg` enabled. In internal tests, v7.5.0 improves the performance by up to 62.5% compared with v6.5.0. [#47757](https://github.com/pingcap/tidb/issues/47757) @[tangenta](https://github.com/tangenta) + ++ TiKV + + - Avoid holding mutex when writing Titan manifest files to prevent affecting other threads [#15351](https://github.com/tikv/tikv/issues/15351) @[Connor1996](https://github.com/Connor1996) + ++ PD + + - Improve the stability and usability of the `evict-slow-trend` scheduler [#7156](https://github.com/tikv/pd/issues/7156) @[LykxSassinato](https://github.com/LykxSassinator) + ++ Tools + + + Backup & Restore (BR) + + - Add a new inter-table backup parameter `table-concurrency` for snapshot backups. This parameter is used to control the inter-table concurrency of meta information such as statistics backup and data validation [#48571](https://github.com/pingcap/tidb/issues/48571) @[3pointer](https://github.com/3pointer) + - During restoring a snapshot backup, BR retries when it encounters certain network errors [#48528](https://github.com/pingcap/tidb/issues/48528) @[Leavrth](https://github.com/Leavrth) + +## Bug fixes + ++ TiDB + + - Prohibit split table operations on non-integer clustered indexes [#47350](https://github.com/pingcap/tidb/issues/47350) @[tangenta](https://github.com/tangenta) + - Fix the issue of encoding time fields with incorrect timezone information [#46033](https://github.com/pingcap/tidb/issues/46033) @[tangenta](https://github.com/tangenta) + - Fix the issue that the Sort operator might cause TiDB to crash during the spill process [#47538](https://github.com/pingcap/tidb/issues/47538) @[windtalker](https://github.com/windtalker) + - Fix the issue that TiDB returns `Can't find column` for queries with `GROUP_CONCAT` [#41957](https://github.com/pingcap/tidb/issues/41957) @[AilinKid](https://github.com/AilinKid) + - Fix the panic issue of `batch-client` in `client-go` [#47691](https://github.com/pingcap/tidb/issues/47691) @[crazycs520](https://github.com/crazycs520) + - Fix the issue of incorrect memory usage estimation in `INDEX_LOOKUP_HASH_JOIN` [#47788](https://github.com/pingcap/tidb/issues/47788) @[SeaRise](https://github.com/SeaRise) + - Fix the issue of uneven workload caused by the rejoining of a TiFlash node that has been offline for a long time [#35418](https://github.com/pingcap/tidb/issues/35418) @[windtalker](https://github.com/windtalker) + - Fix the issue that the chunk cannot be reused when the HashJoin operator performs probe [#48082](https://github.com/pingcap/tidb/issues/48082) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that the `COALESCE()` function returns incorrect result type for `DATE` type parameters [#46475](https://github.com/pingcap/tidb/issues/46475) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that `UPDATE` statements with subqueries are incorrectly converted to PointGet [#48171](https://github.com/pingcap/tidb/issues/48171) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that incorrect results are returned when the cached execution plans contain the comparison between date types and `unix_timestamp` [#48165](https://github.com/pingcap/tidb/issues/48165) @[qw4990](https://github.com/qw4990) + - Fix the issue that an error is reported when default inline common table expressions (CTEs) with aggregate functions or window functions are referenced by recursive CTEs [#47881](https://github.com/pingcap/tidb/issues/47881) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that the optimizer mistakenly selects IndexFullScan to reduce sort introduced by window functions [#46177](https://github.com/pingcap/tidb/issues/46177) @[qw4990](https://github.com/qw4990) + - Fix the issue that multiple references to CTEs result in incorrect results due to condition pushdown of CTEs [#47881](https://github.com/pingcap/tidb/issues/47881) @[winoros](https://github.com/winoros) + - Fix the issue that the MySQL compression protocol cannot handle large loads of data (>=16M) [#47152](https://github.com/pingcap/tidb/issues/47152) [#47157](https://github.com/pingcap/tidb/issues/47157) [#47161](https://github.com/pingcap/tidb/issues/47161) @[dveeden](https://github.com/dveeden) + - Fix the issue that TiDB does not read `cgroup` resource limits when it is started with `systemd` [#47442](https://github.com/pingcap/tidb/issues/47442) @[hawkingrei](https://github.com/hawkingrei) + ++ TiKV + + - Fix the issue that retrying prewrite requests in the pessimistic transaction mode might cause the risk of data inconsistency in rare cases [#11187](https://github.com/tikv/tikv/issues/11187) @[MyonKeminta](https://github.com/MyonKeminta) + ++ PD + + - Fix the issue that `evict-leader-scheduler` might lose configuration [#6897](https://github.com/tikv/pd/issues/6897) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that after a store goes offline, the monitoring metric of its statistics is not deleted [#7180](https://github.com/tikv/pd/issues/7180) @[rleungx](https://github.com/rleungx) + - Fix the issue that `canSync` and `hasMajority` might be calculated incorrectly for clusters adopting the Data Replication Auto Synchronous (DR Auto-Sync) mode when the configuration of Placement Rules is complex [#7201](https://github.com/tikv/pd/issues/7201) @[disksing](https://github.com/disksing) + - Fix the issue that the rule checker does not add Learners according to the configuration of Placement Rules [#7185](https://github.com/tikv/pd/issues/7185) @[nolouch](https://github.com/nolouch) + - Fix the issue that TiDB Dashboard cannot read PD `trace` data correctly [#7253](https://github.com/tikv/pd/issues/7253) @[nolouch](https://github.com/nolouch) + - Fix the issue that PD might panic due to empty Regions obtained internally [#7261](https://github.com/tikv/pd/issues/7261) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that `available_stores` is calculated incorrectly for clusters adopting the Data Replication Auto Synchronous (DR Auto-Sync) mode [#7221](https://github.com/tikv/pd/issues/7221) @[disksing](https://github.com/disksing) + - Fix the issue that PD might delete normal Peers when TiKV nodes are unavailable [#7249](https://github.com/tikv/pd/issues/7249) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that adding multiple TiKV nodes to a large cluster might cause TiKV heartbeat reporting to become slow or stuck [#7248](https://github.com/tikv/pd/issues/7248) @[rleungx](https://github.com/rleungx) + ++ TiFlash + + - Fix the issue that the `UPPER()` and `LOWER()` functions return inconsistent results between TiDB and TiFlash [#7695](https://github.com/pingcap/tiflash/issues/7695) @[windtalker](https://github.com/windtalker) + - Fix the issue that executing queries on empty partitions causes query failure [#8220](https://github.com/pingcap/tiflash/issues/8220) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the panic issue caused by table creation failure when replicating TiFlash replicas [#8217](https://github.com/pingcap/tiflash/issues/8217) @[hongyunyan](https://github.com/hongyunyan) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that PITR might skip restoring the `CREATE INDEX` DDL statement [#47482](https://github.com/pingcap/tidb/issues/47482) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the log backup might get stuck in some scenarios when backing up large wide tables [#15714](https://github.com/tikv/tikv/issues/15714) @[YuJuncen](https://github.com/YuJuncen) + + + TiCDC + + - Fix the performance issue caused by accessing NFS directories when replicating data to an object store sink [#10041](https://github.com/pingcap/tiflow/issues/10041) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that the storage path is misspelled when `claim-check` is enabled [#10036](https://github.com/pingcap/tiflow/issues/10036) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that TiCDC scheduling is not balanced in some cases [#9845](https://github.com/pingcap/tiflow/issues/9845) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that TiCDC might get stuck when replicating data to Kafka [#9855](https://github.com/pingcap/tiflow/issues/9855) @[hicqu](https://github.com/hicqu) + - Fix the issue that the TiCDC processor might panic in some cases [#9849](https://github.com/pingcap/tiflow/issues/9849) [#9915](https://github.com/pingcap/tiflow/issues/9915) @[hicqu](https://github.com/hicqu) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that enabling `kv-client.enable-multiplexing` causes replication tasks to get stuck [#9673](https://github.com/pingcap/tiflow/issues/9673) @[fubinzh](https://github.com/fubinzh) + - Fix the issue that an owner node gets stuck due to NFS failure when the redo log is enabled [#9886](https://github.com/pingcap/tiflow/issues/9886) @[3AceShowHand](https://github.com/3AceShowHand) + +## Performance test + +To learn about the performance of TiDB v7.5.0, you can refer to the [TPC-C performance test report](https://docs.pingcap.com/tidbcloud/v7.5.0-performance-benchmarking-with-tpcc) and [Sysbench performance test report](https://docs.pingcap.com/tidbcloud/v7.5.0-performance-benchmarking-with-sysbench) of the TiDB Cloud Dedicated cluster. + +## Contributors + +We would like to thank the following contributors from the TiDB community: + +- [jgrande](https://github.com/jgrande) (First-time contributor) +- [shawn0915](https://github.com/shawn0915) diff --git a/releases/release-7.5.1.md b/releases/release-7.5.1.md new file mode 100644 index 0000000000000..8a50ba60a992c --- /dev/null +++ b/releases/release-7.5.1.md @@ -0,0 +1,233 @@ +--- +title: TiDB 7.5.1 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 7.5.1. +--- + +# TiDB 7.5.1 Release Notes + +Release date: February 29, 2024 + +TiDB version: 7.5.1 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.5/production-deployment-using-tiup) + +## Compatibility changes + +- Prohibit setting [`require_secure_transport`](https://docs.pingcap.com/tidb/v7.5/system-variables#require_secure_transport-new-in-v610) to `ON` in Security Enhanced Mode (SEM) to prevent potential connectivity issues for users [#47665](https://github.com/pingcap/tidb/issues/47665) @[tiancaiamao](https://github.com/tiancaiamao) +- To reduce the overhead of log printing, TiFlash changes the default value of `logger.level` from `"debug"` to `"info"` [#8641](https://github.com/pingcap/tiflash/issues/8641) @[JaySon-Huang](https://github.com/JaySon-Huang) +- Introduce the TiKV configuration item [`gc.num-threads`](https://docs.pingcap.com/tidb/v7.5/tikv-configuration-file#num-threads-new-in-v658-and-v751) to set the number of GC threads when `enable-compaction-filter` is `false` [#16101](https://github.com/tikv/tikv/issues/16101) @[tonyxuqqi](https://github.com/tonyxuqqi) +- TiCDC Changefeed introduces the following new configuration items: + - [`compression`](/ticdc/ticdc-changefeed-config.md): enables you to configure the compression behavior of redo log files [#10176](https://github.com/pingcap/tiflow/issues/10176) @[sdojjy](https://github.com/sdojjy) + - [`sink.cloud-storage-config`](/ticdc/ticdc-changefeed-config.md): enables you to set the automatic cleanup of historical data when replicating data to object storage [#10109](https://github.com/pingcap/tiflow/issues/10109) @[CharlesCheung96](https://github.com/CharlesCheung96) + - [`consistent.flush-concurrency`](/ticdc/ticdc-changefeed-config.md): enables you to set the concurrency for uploading a single redo file [#10226](https://github.com/pingcap/tiflow/issues/10226) @[sdojjy](https://github.com/sdojjy) + +## Improvements + ++ TiDB + + - Use `tikv_client_read_timeout` during the DDL schema reload process to reduce the impact of Meta Region Leader read unavailability on the cluster [#48124](https://github.com/pingcap/tidb/issues/48124) @[cfzjywxk](https://github.com/cfzjywxk) + - Enhance observability related to resource control [#49318](https://github.com/pingcap/tidb/issues/49318) @[glorv](https://github.com/glorv) @[bufferflies](https://github.com/bufferflies) @[nolouch](https://github.com/nolouch) + + As more and more users use resource groups to isolate application workloads, Resource Control provides enhanced data based on resource groups. This helps you monitor resource group workloads and settings, ensuring that you can quickly identify and accurately diagnose problems, including: + + - [Slow Queries](/identify-slow-queries.md): add the resource group name, resource unit (RU) consumption, and time for waiting for resources. + - [Statement Summary Tables](/statement-summary-tables.md): add the resource group name, RU consumption, and time for waiting for resources. + - In the system variable [`tidb_last_query_info`](/system-variables.md#tidb_last_query_info-new-in-v4014), add a new entry `ru_consumption` to indicate the consumed [RU](/tidb-resource-control.md#what-is-request-unit-ru) by SQL statements. You can use this variable to get the resource consumption of the last statement in the session. + - Add database metrics based on resource groups: QPS/TPS, execution time (P999/P99/P95), number of failures, and number of connections. + + - Modify the `CANCEL IMPORT JOB` statement to a synchronous statement [#48736](https://github.com/pingcap/tidb/issues/48736) @[D3Hunter](https://github.com/D3Hunter) + - Support the [`FLASHBACK CLUSTER TO TSO`](https://docs.pingcap.com/tidb/v7.5/sql-statement-flashback-cluster) syntax [#48372](https://github.com/pingcap/tidb/issues/48372) @[BornChanger](https://github.com/BornChanger) + - Optimize the TiDB implementation when handling some type conversions and fix related issues [#47945](https://github.com/pingcap/tidb/issues/47945) [#47864](https://github.com/pingcap/tidb/issues/47864) [#47829](https://github.com/pingcap/tidb/issues/47829) [#47816](https://github.com/pingcap/tidb/issues/47816) @[YangKeao](https://github.com/YangKeao) @[lcwangchao](https://github.com/lcwangchao) + - When a non-binary collation is set and the query includes `LIKE`, the optimizer generates an `IndexRangeScan` to improve the execution efficiency [#48181](https://github.com/pingcap/tidb/issues/48181) [#49138](https://github.com/pingcap/tidb/issues/49138) @[time-and-fate](https://github.com/time-and-fate) + - Enhance the ability to convert `OUTER JOIN` to `INNER JOIN` in specific scenarios [#49616](https://github.com/pingcap/tidb/issues/49616) @[qw4990](https://github.com/qw4990) + - Support multiple accelerated `ADD INDEX` DDL tasks to be queued for execution, instead of falling back to normal `ADD INDEX` tasks [#47758](https://github.com/pingcap/tidb/issues/47758) @[tangenta](https://github.com/tangenta) + - Improve the speed of adding indexes to empty tables [#49682](https://github.com/pingcap/tidb/issues/49682) @[zimulala](https://github.com/zimulala) + ++ TiKV + + - Enhance the slow store detection algorithm by improving its sensitivity and reducing the false-positive rate, especially in intensive read and write load scenarios [#15909](https://github.com/tikv/tikv/issues/15909) @[LykxSassinator](https://github.com/LykxSassinator) + ++ TiFlash + + - Improve the calculation method for [Request Unit (RU)](/tidb-resource-control.md#what-is-request-unit-ru) to make RU values more stable [#8391](https://github.com/pingcap/tiflash/issues/8391) @[guo-shaoge](https://github.com/guo-shaoge) + - Reduce the impact of disk performance jitter on read latency [#8583](https://github.com/pingcap/tiflash/issues/8583) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Reduce the impact of background GC tasks on read and write task latency [#8650](https://github.com/pingcap/tiflash/issues/8650) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - Improve the speed of merging SST files during data restore by using a more efficient algorithm [#50613](https://github.com/pingcap/tidb/issues/50613) @[Leavrth](https://github.com/Leavrth) + - Support creating databases in batch during data restore [#50767](https://github.com/pingcap/tidb/issues/50767) @[Leavrth](https://github.com/Leavrth) + - Support ingesting SST files in batch during data restore [#16267](https://github.com/tikv/tikv/issues/16267) @[3pointer](https://github.com/3pointer) + - Print the information of the slowest Region that affects global checkpoint advancement in logs and metrics during log backups [#51046](https://github.com/pingcap/tidb/issues/51046) @[YuJuncen](https://github.com/YuJuncen) + - Improve the table creation performance of the `RESTORE` statement in scenarios with large datasets [#48301](https://github.com/pingcap/tidb/issues/48301) @[Leavrth](https://github.com/Leavrth) + - BR can pause Region merging by setting the `merge-schedule-limit` configuration to `0` [#7148](https://github.com/tikv/pd/issues/7148) @[BornChanger](https://github.com/3pointer) + - Refactor the BR exception handling mechanism to increase tolerance for unknown errors [#47656](https://github.com/pingcap/tidb/issues/47656) @[3pointer](https://github.com/3pointer) + + + TiCDC + + - Support searching TiCDC logs in the TiDB Dashboard [#10263](https://github.com/pingcap/tiflow/issues/10263) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Support [querying the downstream synchronization status of a changefeed](https://docs.pingcap.com/tidb/v7.5/ticdc-open-api-v2#query-whether-a-specific-replication-task-is-completed), which helps you determine whether the upstream data changes received by TiCDC have been synchronized to the downstream system completely [#10289](https://github.com/pingcap/tiflow/issues/10289) @[hongyunyan](https://github.com/hongyunyan) + - Improve the performance of TiCDC replicating data to object storage by increasing parallelism [#10098](https://github.com/pingcap/tiflow/issues/10098) @[CharlesCheung96](https://github.com/CharlesCheung96) + + + TiDB Lightning + + - Improve the performance of `ALTER TABLE` when importing a large number of small tables [#50105](https://github.com/pingcap/tidb/issues/50105) @[D3Hunter](https://github.com/D3Hunter) + +## Bug fixes + ++ TiDB + + - Fix the issue that setting the system variable `tidb_service_scope` does not take effect [#49245](https://github.com/pingcap/tidb/issues/49245) @[ywqzzy](https://github.com/ywqzzy) + - Fix the issue that the communication protocol cannot handle packets larger than or equal to 16 MB when compression is enabled [#47157](https://github.com/pingcap/tidb/issues/47157) [#47161](https://github.com/pingcap/tidb/issues/47161) @[dveeden](https://github.com/dveeden) + - Fix the issue that the `approx_percentile` function might cause TiDB panic [#40463](https://github.com/pingcap/tidb/issues/40463) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that TiDB might implicitly insert the `from_binary` function when the argument of a string function is a `NULL` constant, causing some expressions unable to be pushed down to TiFlash [#49526](https://github.com/pingcap/tidb/issues/49526) @[YangKeao](https://github.com/YangKeao) + - Fix the goroutine leak issue that might occur when the `HashJoin` operator fails to spill to disk [#50841](https://github.com/pingcap/tidb/issues/50841) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that `BIT` type columns might cause query errors due to decode failures when they are involved in calculations of some functions [#49566](https://github.com/pingcap/tidb/issues/49566) [#50850](https://github.com/pingcap/tidb/issues/50850) [#50855](https://github.com/pingcap/tidb/issues/50855) @[jiyfhust](https://github.com/jiyfhust) + - Fix the goroutine leak issue that occurs when the memory usage of CTE queries exceed limits [#50337](https://github.com/pingcap/tidb/issues/50337) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that wrong results might be returned when TiFlash late materialization processes associated columns [#49241](https://github.com/pingcap/tidb/issues/49241) [#51204](https://github.com/pingcap/tidb/issues/51204) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that the background job thread of TiDB might panic when TiDB records historical statistics [#49076](https://github.com/pingcap/tidb/issues/49076) @[hawkingrei](https://github.com/hawkingrei) + - Fix the error that might occur when TiDB merges histograms of global statistics for partitioned tables [#49023](https://github.com/pingcap/tidb/issues/49023) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the historical statistics of the `stats_meta` table are not updated after a partition is dropped [#49334](https://github.com/pingcap/tidb/issues/49334) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue of incorrect query results caused by multi-valued indexes mistakenly selected as the `Index Join` probe side [#50382](https://github.com/pingcap/tidb/issues/50382) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that the `USE_INDEX_MERGE` hint does not take effect on multi-valued indexes [#50553](https://github.com/pingcap/tidb/issues/50553) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that users might get errors when querying the `INFORMATION_SCHEMA.ANALYZE_STATUS` system table [#48835](https://github.com/pingcap/tidb/issues/48835) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue of wrong query results due to TiDB incorrectly eliminating constant values in `group by` [#38756](https://github.com/pingcap/tidb/issues/38756) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that the `processed_rows` of the `ANALYZE` task on a table might exceed the total number of rows in that table [#50632](https://github.com/pingcap/tidb/issues/50632) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that TiDB might panic when using the `EXECUTE` statement to execute `PREPARE STMT` after the `tidb_enable_prepared_plan_cache` system variable is enabled and then disabled [#49344](https://github.com/pingcap/tidb/issues/49344) @[qw4990](https://github.com/qw4990) + - Fix the `Column ... in from clause is ambiguous` error that might occur when a query uses `NATURAL JOIN` [#32044](https://github.com/pingcap/tidb/issues/32044) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that using a multi-valued index to access an empty JSON array might return incorrect results [#50125](https://github.com/pingcap/tidb/issues/50125) @[YangKeao](https://github.com/YangKeao) + - Fix the `Can't find column ...` error that might occur when aggregate functions are used for group calculations [#50926](https://github.com/pingcap/tidb/issues/50926) @[qw4990](https://github.com/qw4990) + - Fix the issue that the control of `SET_VAR` for variables of the string type might become invalid [#50507](https://github.com/pingcap/tidb/issues/50507) @[qw4990](https://github.com/qw4990) + - Fix the issue that high CPU usage of TiDB occurs due to long-term memory pressure caused by `tidb_server_memory_limit` [#48741](https://github.com/pingcap/tidb/issues/48741) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that the completion times of two DDL tasks with dependencies are incorrectly sequenced [#49498](https://github.com/pingcap/tidb/issues/49498) @[tangenta](https://github.com/tangenta) + - Fix the issue that illegal optimizer hints might cause valid hints to be ineffective [#49308](https://github.com/pingcap/tidb/issues/49308) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that DDL statements with the `CHECK` constraint are stuck [#47632](https://github.com/pingcap/tidb/issues/47632) @[jiyfhust](https://github.com/jiyfhust) + - Fix the issue that the behavior of the `ENFORCED` option in the `CHECK` constraint is inconsistent with MySQL 8.0 [#47567](https://github.com/pingcap/tidb/issues/47567) [#47631](https://github.com/pingcap/tidb/issues/47631) @[jiyfhust](https://github.com/jiyfhust) + - Fix the issue that CTE queries might report an error `type assertion for CTEStorageMap failed` during the retry process [#46522](https://github.com/pingcap/tidb/issues/46522) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the `DELETE` and `UPDATE` statements using index lookup might report an error when `tidb_multi_statement_mode` mode is enabled [#50012](https://github.com/pingcap/tidb/issues/50012) @[tangenta](https://github.com/tangenta) + - Fix the issue that `UPDATE` or `DELETE` statements containing `WITH RECURSIVE` CTEs might produce incorrect results [#48969](https://github.com/pingcap/tidb/issues/48969) @[winoros](https://github.com/winoros) + - Fix the issue that the optimizer incorrectly converts TiFlash selection path to the DUAL table in specific scenarios [#49285](https://github.com/pingcap/tidb/issues/49285) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that the same query plan has different `PLAN_DIGEST` values in some cases [#47634](https://github.com/pingcap/tidb/issues/47634) @[King-Dylan](https://github.com/King-Dylan) + - Fix the issue that after the time window for automatic statistics updates is configured, statistics might still be updated outside that time window [#49552](https://github.com/pingcap/tidb/issues/49552) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the query result is incorrect when an `ENUM` type column is used as the join key [#48991](https://github.com/pingcap/tidb/issues/48991) @[winoros](https://github.com/winoros) + - Fix the issue that executing `UNIQUE` index lookup with an `ORDER BY` clause might cause an error [#49920](https://github.com/pingcap/tidb/issues/49920) @[jackysp](https://github.com/jackysp) + - Fix the issue that `LIMIT` in multi-level nested `UNION` queries might become ineffective [#49874](https://github.com/pingcap/tidb/issues/49874) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that the result of `COUNT(INT)` calculated by MPP might be incorrect [#48643](https://github.com/pingcap/tidb/issues/48643) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that parsing invalid values of `ENUM` or `SET` types would directly cause SQL statement errors [#49487](https://github.com/pingcap/tidb/issues/49487) @[winoros](https://github.com/winoros) + - Fix the issue that TiDB panics and reports an error `invalid memory address or nil pointer dereference` [#42739](https://github.com/pingcap/tidb/issues/42739) @[CbcWestwolf](https://github.com/CbcWestwolf) + - Fix the issue that executing `UNION ALL` with the DUAL table as the first subnode might cause an error [#48755](https://github.com/pingcap/tidb/issues/48755) @[winoros](https://github.com/winoros) + - Fix the issue that common hints do not take effect in `UNION ALL` statements [#50068](https://github.com/pingcap/tidb/issues/50068) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that TiDB server might panic during graceful shutdown [#36793](https://github.com/pingcap/tidb/issues/36793) @[bb7133](https://github.com/bb7133) + - Fix the issue that Daylight Saving Time is displayed incorrectly in some time zones [#49586](https://github.com/pingcap/tidb/issues/49586) @[overvenus](https://github.com/overvenus) + - Fix the issue that static `CALIBRATE RESOURCE` relies on the Prometheus data [#49174](https://github.com/pingcap/tidb/issues/49174) @[glorv](https://github.com/glorv) + - Fix the issue that hints cannot be used in `REPLACE INTO` statements [#34325](https://github.com/pingcap/tidb/issues/34325) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that executing queries containing the `GROUP_CONCAT(ORDER BY)` syntax might return errors [#49986](https://github.com/pingcap/tidb/issues/49986) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that TiDB server might consume a significant amount of resources when the enterprise plugin for audit logging is used [#49273](https://github.com/pingcap/tidb/issues/49273) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that using old interfaces might cause inconsistent metadata for tables [#49751](https://github.com/pingcap/tidb/issues/49751) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that disabling `tidb_enable_collect_execution_info` causes the coprocessor cache to panic [#48212](https://github.com/pingcap/tidb/issues/48212) @[you06](https://github.com/you06) + - Fix the issue that executing `ALTER TABLE ... LAST PARTITION` fails when the partition column type is `DATETIME` [#48814](https://github.com/pingcap/tidb/issues/48814) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that the `COMMIT` or `ROLLBACK` operation executed through `COM_STMT_EXECUTE` fails to terminate transactions that have timed out [#49151](https://github.com/pingcap/tidb/issues/49151) @[zyguan](https://github.com/zyguan) + - Fix the issue that histogram statistics might not be parsed into readable strings when the histogram boundary contains `NULL` [#49823](https://github.com/pingcap/tidb/issues/49823) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that queries containing common table expressions (CTEs) unexpectedly get stuck when the memory limit is exceeded [#49096](https://github.com/pingcap/tidb/issues/49096) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that data is inconsistent under the TiDB Distributed eXecution Framework (DXF) when executing `ADD INDEX` after the DDL Owner is network isolated [#49773](https://github.com/pingcap/tidb/issues/49773) @[tangenta](https://github.com/tangenta) + - Fix the issue that the auto-increment ID allocation reports an error due to concurrent conflicts when using an auto-increment column with `AUTO_ID_CACHE=1` [#50519](https://github.com/pingcap/tidb/issues/50519) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that TiDB might panic when a query contains the Apply operator and the `fatal error: concurrent map writes` error occurs [#50347](https://github.com/pingcap/tidb/issues/50347) @[SeaRise](https://github.com/SeaRise) + - Fix the TiDB node panic issue that occurs when DDL `jobID` is restored to 0 [#46296](https://github.com/pingcap/tidb/issues/46296) @[jiyfhust](https://github.com/jiyfhust) + - Fix the issue that query results are incorrect due to `STREAM_AGG()` incorrectly handling CI [#49902](https://github.com/pingcap/tidb/issues/49902) @[wshwsh12](https://github.com/wshwsh12) + - Mitigate the issue that TiDB nodes might encounter OOM errors when dealing with a large number of tables or partitions [#50077](https://github.com/pingcap/tidb/issues/50077) @[zimulala](https://github.com/zimulala) + - Fix the issue that the `LEADING` hint does not take effect in `UNION ALL` statements [#50067](https://github.com/pingcap/tidb/issues/50067) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that `LIMIT` and `OPRDERBY` might be invalid in nested `UNION` queries [#49377](https://github.com/pingcap/tidb/issues/49377) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that a query containing the IndexHashJoin operator gets stuck when memory exceeds `tidb_mem_quota_query` [#49033](https://github.com/pingcap/tidb/issues/49033) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that TiDB returns wrong query results when processing `ENUM` or `SET` types by constant propagation [#49440](https://github.com/pingcap/tidb/issues/49440) @[winoros](https://github.com/winoros) + - Fix the issue that executing `SELECT INTO OUTFILE` using the `PREPARE` method incorrectly returns a success message instead of an error [#49166](https://github.com/pingcap/tidb/issues/49166) @[qw4990](https://github.com/qw4990) + - Fix the issue that enforced sorting might become ineffective when a query uses optimizer hints (such as `STREAM_AGG()`) that enforce sorting and its execution plan contains `IndexMerge` [#49605](https://github.com/pingcap/tidb/issues/49605) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that tables with `AUTO_ID_CACHE=1` might lead to gRPC client leaks when there are a large number of tables [#48869](https://github.com/pingcap/tidb/issues/48869) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that in non-strict mode (`sql_mode = ''`), truncation during executing `INSERT` still reports an error [#49369](https://github.com/pingcap/tidb/issues/49369) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that using the `_` wildcard in `LIKE` when the data contains trailing spaces can result in incorrect query results [#48983](https://github.com/pingcap/tidb/issues/48983) @[time-and-fate](https://github.com/time-and-fate) + - Fix the issue that executing `ADMIN CHECK` after updating the `tidb_mem_quota_query` system variable returns `ERROR 8175` [#49258](https://github.com/pingcap/tidb/issues/49258) @[tangenta](https://github.com/tangenta) + - Fix the issue of excessive statistical error in constructing statistics caused by Golang's implicit conversion algorithm [#49801](https://github.com/pingcap/tidb/issues/49801) @[qw4990](https://github.com/qw4990) + - Fix the issue that queries containing CTEs report `runtime error: index out of range [32] with length 32` when `tidb_max_chunk_size` is set to a small value [#48808](https://github.com/pingcap/tidb/issues/48808) @[guo-shaoge](https://github.com/guo-shaoge) + ++ TiKV + + - Fix the issue that enabling `tidb_enable_row_level_checksum` might cause TiKV to panic [#16371](https://github.com/tikv/tikv/issues/16371) @[cfzjywxk](https://github.com/cfzjywxk) + - Fix the issue that TiKV might panic when gRPC threads are checking `is_shutdown` [#16236](https://github.com/tikv/tikv/issues/16236) @[pingyu](https://github.com/pingyu) + - Fix the issue that TiKV converts the time zone incorrectly for Brazil and Egypt [#16220](https://github.com/tikv/tikv/issues/16220) @[overvenus](https://github.com/overvenus) + - Fix the issue that `blob-run-mode` in Titan cannot be updated online [#15978](https://github.com/tikv/tikv/issues/15978) @[tonyxuqqi](https://github.com/tonyxuqqi) + - Fix the issue that TiDB and TiKV might produce inconsistent results when processing `DECIMAL` arithmetic multiplication truncation [#16268](https://github.com/tikv/tikv/issues/16268) @[solotzg](https://github.com/solotzg) + - Fix the issue that Flashback might get stuck when encountering `notLeader` or `regionNotFound` [#15712](https://github.com/tikv/tikv/issues/15712) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that damaged SST files might be spread to other TiKV nodes [#15986](https://github.com/tikv/tikv/issues/15986) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that if TiKV runs extremely slowly, it might panic after Region merge [#16111](https://github.com/tikv/tikv/issues/16111) @[overvenus](https://github.com/overvenus) + - Fix the issue that the joint state of DR Auto-Sync might time out when scaling out [#15817](https://github.com/tikv/tikv/issues/15817) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that Resolved TS might be blocked for two hours [#11847](https://github.com/tikv/tikv/issues/11847) [#15520](https://github.com/tikv/tikv/issues/15520) [#39130](https://github.com/pingcap/tidb/issues/39130) @[overvenus](https://github.com/overvenus) + - Fix the issue that `cast_duration_as_time` might return incorrect results [#16211](https://github.com/tikv/tikv/issues/16211) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that TiKV hangs in corner cases (for example, when disk I/O operations are blocked), which affects availability [#16368](https://github.com/tikv/tikv/issues/16368) @[LykxSassinator](https://github.com/LykxSassinator) + ++ PD + + - Fix the issue that querying resource groups in batch might cause PD to panic [#7206](https://github.com/tikv/pd/issues/7206) @[nolouch](https://github.com/nolouch) + - Fix the issue that PD cannot read resource limitations when it is started with `systemd` [#7628](https://github.com/tikv/pd/issues/7628) @[bufferflies](https://github.com/bufferflies) + - Fix the issue that continuous jitter in PD disk latency might cause PD to fail to select a new leader [#7251](https://github.com/tikv/pd/issues/7251) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that a network partition in PD might cause scheduling not to be started immediately [#7016](https://github.com/tikv/pd/issues/7016) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that the PD monitoring item `learner-peer-count` does not synchronize the old value after a leader switch [#7728](https://github.com/tikv/pd/issues/7728) @[CabinfeverB](https://github.com/CabinfeverB) + - Fix the issue that when PD leader is transferred and there is a network partition between the new leader and the PD client, the PD client fails to update the information of the leader [#7416](https://github.com/tikv/pd/issues/7416) @[CabinfeverB](https://github.com/CabinfeverB) + - Fix some security issues by upgrading the version of Gin Web Framework from v1.8.1 to v1.9.1 [#7438](https://github.com/tikv/pd/issues/7438) @[niubell](https://github.com/niubell) + - Fix the issue that the orphan peer is deleted when the number of replicas does not meet the requirements [#7584](https://github.com/tikv/pd/issues/7584) @[bufferflies](https://github.com/bufferflies) + - Fix the issue that querying a Region without a leader using `pd-ctl` might cause PD to panic [#7630](https://github.com/tikv/pd/issues/7630) @[rleungx](https://github.com/rleungx) + ++ TiFlash + + - Fix the issue that TiFlash might panic due to unstable network connections with PD during replica migration [#8323](https://github.com/pingcap/tiflash/issues/8323) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that removing and then re-adding TiFlash replicas might lead to data corruption in TiFlash [#8695](https://github.com/pingcap/tiflash/issues/8695) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix a potential issue that `FLASHBACK TABLE` or `RECOVER TABLE` might fail to recover data of some TiFlash replicas if `DROP TABLE` is executed immediately after data insertion [#8395](https://github.com/pingcap/tiflash/issues/8395) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix incorrect display of maximum percentile time for some panels in Grafana [#8076](https://github.com/pingcap/tiflash/issues/8076) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash might crash during remote reads [#8685](https://github.com/pingcap/tiflash/issues/8685) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that TiFlash incorrectly handles `ENUM` when the `ENUM` value is 0 [#8311](https://github.com/pingcap/tiflash/issues/8311) @[solotzg](https://github.com/solotzg) + - Fix the issue that short queries executed successfully print excessive info logs [#8592](https://github.com/pingcap/tiflash/issues/8592) @[windtalker](https://github.com/windtalker) + - Fix the issue that the memory usage increases significantly due to slow queries [#8564](https://github.com/pingcap/tiflash/issues/8564) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that the `lowerUTF8` and `upperUTF8` functions do not allow characters in different cases to occupy different bytes [#8484](https://github.com/pingcap/tiflash/issues/8484) @[gengliqi](https://github.com/gengliqi) + - Fix the potential OOM issue that might occur when scanning multiple partitioned tables during stream read [#8505](https://github.com/pingcap/tiflash/issues/8505) @[gengliqi](https://github.com/gengliqi) + - Fix the issue of memory leak when TiFlash encounters memory limitation during query [#8447](https://github.com/pingcap/tiflash/issues/8447) @[JinheLin](https://github.com/JinheLin) + - Fix the TiFlash panic issue when TiFlash encounters conflicts during concurrent DDL execution [#8578](https://github.com/pingcap/tiflash/issues/8578) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash panics after executing `ALTER TABLE ... MODIFY COLUMN ... NOT NULL`, which changes nullable columns to non-nullable [#8419](https://github.com/pingcap/tiflash/issues/8419) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that query results are incorrect when querying with filtering conditions like `ColumnRef in (Literal, Func...)` [#8631](https://github.com/pingcap/tiflash/issues/8631) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that data of TiFlash replicas would still be garbage collected after executing `FLASHBACK DATABASE` [#8450](https://github.com/pingcap/tiflash/issues/8450) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash might not be able to select the GC owner of object storage data under the disaggregated storage and compute architecture [#8519](https://github.com/pingcap/tiflash/issues/8519) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the random invalid memory access issue that might occur with `GREATEST` or `LEAST` functions containing constant string parameters [#8604](https://github.com/pingcap/tiflash/issues/8604) @[windtalker](https://github.com/windtalker) + - Fix the issue that TiFlash replica data might be accidentally deleted after performing point-in-time recovery (PITR) or executing `FLASHBACK CLUSTER TO`, which might result in data anomalies [#8777](https://github.com/pingcap/tiflash/issues/8777) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash Anti Semi Join might return incorrect results when the join includes non-equivalent conditions [#8791](https://github.com/pingcap/tiflash/issues/8791) @[windtalker](https://github.com/windtalker) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that data restore is slowed down due to absence of a leader on a TiKV node [#50566](https://github.com/pingcap/tidb/issues/50566) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that full restore still requires the target cluster to be empty after the `--filter` option is specified [#51009](https://github.com/pingcap/tidb/issues/51009) @[3pointer](https://github.com/3pointer) + - Fix the issue that when resuming from a checkpoint after data restore fails, an error `the target cluster is not fresh` occurs [#50232](https://github.com/pingcap/tidb/issues/50232) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that stopping a log backup task causes TiDB to crash [#50839](https://github.com/pingcap/tidb/issues/50839) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that the `Unsupported collation` error is reported when you restore data from backups of an old version [#49466](https://github.com/pingcap/tidb/issues/49466) @[3pointer](https://github.com/3pointer) + - Fix the issue that the log backup task can start but does not work properly if failing to connect to PD during task initialization [#16056](https://github.com/tikv/tikv/issues/16056) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that BR generates incorrect URIs for external storage files [#48452](https://github.com/pingcap/tidb/issues/48452) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that log backup gets stuck after changing the TiKV IP address on the same node [#50445](https://github.com/pingcap/tidb/issues/50445) @[3pointer](https://github.com/3pointer) + - Fix the issue that BR cannot retry when encountering an error while reading file content from S3 [#49942](https://github.com/pingcap/tidb/issues/49942) @[Leavrth](https://github.com/Leavrth) + + + TiCDC + + - Fix the issue that the sink module fails to restart correctly after encountering an error when Syncpoint is enabled (`enable-sync-point = true`) [#10091](https://github.com/pingcap/tiflow/issues/10091) @[hicqu](https://github.com/hicqu) + - Fix the issue that the file sequence number generated by the storage service might not increment correctly when using the storage sink [#10352](https://github.com/pingcap/tiflow/issues/10352) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that the Syncpoint table might be incorrectly replicated [#10576](https://github.com/pingcap/tiflow/issues/10576) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that OAuth2.0, TLS, and mTLS cannot be enabled properly when using Apache Pulsar as the downstream [#10602](https://github.com/pingcap/tiflow/issues/10602) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that TiCDC returns the `ErrChangeFeedAlreadyExists` error when concurrently creating multiple changefeeds [#10430](https://github.com/pingcap/tiflow/issues/10430) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that the changefeed `resolved ts` does not advance in extreme cases [#10157](https://github.com/pingcap/tiflow/issues/10157) @[sdojjy](https://github.com/sdojjy) + - Fix the issue that TiCDC mistakenly closes the connection with TiKV in certain special scenarios [#10239](https://github.com/pingcap/tiflow/issues/10239) @[hicqu](https://github.com/hicqu) + - Fix the issue that the TiCDC server might panic when replicating data to an object storage service [#10137](https://github.com/pingcap/tiflow/issues/10137) @[sdojjy](https://github.com/sdojjy) + - Fix the issue that the changefeed reports an error after `TRUNCATE PARTITION` is executed on the upstream table [#10522](https://github.com/pingcap/tiflow/issues/10522) @[sdojjy](https://github.com/sdojjy) + - Fix the issue that after filtering out `add table partition` events is configured in `ignore-event`, TiCDC does not replicate other types of DML changes for related partitions to the downstream [#10524](https://github.com/pingcap/tiflow/issues/10524) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the potential data race issue during `kv-client` initialization [#10095](https://github.com/pingcap/tiflow/issues/10095) @[3AceShowHand](https://github.com/3AceShowHand) + + + TiDB Data Migration (DM) + + - Fix the issue that a migration task error occurs when the downstream table structure contains `shard_row_id_bits` [#10308](https://github.com/pingcap/tiflow/issues/10308) @[GMHDBJD](https://github.com/GMHDBJD) + - Fix the issue that DM encounters "event type truncate not valid" error that causes the upgrade to fail [#10282](https://github.com/pingcap/tiflow/issues/10282) @[GMHDBJD](https://github.com/GMHDBJD) diff --git a/releases/release-7.5.2.md b/releases/release-7.5.2.md new file mode 100644 index 0000000000000..36baba41161ed --- /dev/null +++ b/releases/release-7.5.2.md @@ -0,0 +1,253 @@ +--- +title: TiDB 7.5.2 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 7.5.2. +--- + +# TiDB 7.5.2 Release Notes + +Release date: June 13, 2024 + +TiDB version: 7.5.2 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.5/production-deployment-using-tiup) + +## Compatibility changes + +- Add a TiKV configuration item [`track-and-verify-wals-in-manifest`](https://docs.pingcap.com/tidb/v7.5/tikv-configuration-file#track-and-verify-wals-in-manifest-new-in-v659-v715-and-v752) for RocksDB, which helps you investigate possible corruption of Write Ahead Log (WAL) [#16549](https://github.com/tikv/tikv/issues/16549) @[v01dstar](https://github.com/v01dstar) +- Must set the line terminator when using TiDB Lightning `strict-format` or `SPLIT_FILE` to import CSV files [#37338](https://github.com/pingcap/tidb/issues/37338) @[lance6716](https://github.com/lance6716) +- Add the `sink.open.output-old-value` configuration item for TiCDC Open Protocol to control whether to output the value before the update to the downstream [#10916](https://github.com/pingcap/tiflow/issues/10916) @[sdojjy](https://github.com/sdojjy) +- In earlier versions, when processing a transaction containing `UPDATE` changes, if the primary key or non-null unique index value is modified in an `UPDATE` event, TiCDC splits this event into `DELETE` and `INSERT` events. Starting from v7.5.2, when using the MySQL sink, TiCDC splits an `UPDATE` event into `DELETE` and `INSERT` events if the transaction `commitTS` for the `UPDATE` change is less than TiCDC `thresholdTS` (which is the current timestamp fetched from PD when TiCDC starts replicating the corresponding table to the downstream). This behavior change addresses the issue of downstream data inconsistencies caused by the potentially incorrect order of `UPDATE` events received by TiCDC, which can lead to an incorrect order of split `DELETE` and `INSERT` events. For more information, see [documentation](https://docs.pingcap.com/tidb/v7.5/ticdc-split-update-behavior#split-update-events-for-mysql-sinks). [#10918](https://github.com/pingcap/tiflow/issues/10918) @[lidezhu](https://github.com/lidezhu) + +## Improvements + ++ TiDB + + - Optimize the issue that the `ANALYZE` statement blocks the metadata lock [#47475](https://github.com/pingcap/tidb/issues/47475) @[wjhuang2016](https://github.com/wjhuang2016) + - Improve the MySQL compatibility of expression default values displayed in the output of `SHOW CREATE TABLE` [#52939](https://github.com/pingcap/tidb/issues/52939) @[CbcWestwolf](https://github.com/CbcWestwolf) + - Enhance the handling of DNF items that are always `false` by directly ignoring such filter conditions, thus avoiding unnecessary full table scans [#40997](https://github.com/pingcap/tidb/issues/40997) @[hi-rustin](https://github.com/Rustin170506) + - Optimize the statistics for the execution process of the TiFlash `TableScan` operator in `EXPLAIN ANALYZE` [#51727](https://github.com/pingcap/tidb/issues/51727) @[JinheLin](https://github.com/JinheLin) + - Remove stores without Regions during MPP load balancing [#52313](https://github.com/pingcap/tidb/issues/52313) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Support loading Regions in batch from PD to speed up the conversion process from the KV range to Regions when querying large tables [#51326](https://github.com/pingcap/tidb/issues/51326) @[SeaRise](https://github.com/SeaRise) + - On the `Resource Control` monitoring page, add a new panel `RU(Max)` to show the maximum RU consumption rate for each resource group [#49318](https://github.com/pingcap/tidb/issues/49318) @[nolouch](https://github.com/nolouch) + - Improve sync load performance to reduce latency in loading statistics [#52994](https://github.com/pingcap/tidb/issues/52294) [hawkingrei](https://github.com/hawkingrei) + - Increase concurrency of statistics initialization to speed up startup [#52466](https://github.com/pingcap/tidb/issues/52466) [#52102](https://github.com/pingcap/tidb/issues/52102) [#52553](https://github.com/pingcap/tidb/issues/52553) [hawkingrei](https://github.com/hawkingrei) + ++ TiKV + + - Adjust the log level of coprocessor errors from `warn` to `debug` to reduce unnecessary logs of the cluster [#15881](https://github.com/tikv/tikv/issues/15881) @[cfzjywxk](https://github.com/cfzjywxk) + - Add monitoring metrics for the queue time for processing CDC events to facilitate troubleshooting downstream CDC event latency issues [#16282](https://github.com/tikv/tikv/issues/16282) @[hicqu](https://github.com/hicqu) + - Avoid performing IO operations on snapshot files in raftstore threads to improve TiKV stability [#16564](https://github.com/tikv/tikv/issues/16564) @[Connor1996](https://github.com/Connor1996) + - Add slow logs for peer and store messages [#16600](https://github.com/tikv/tikv/issues/16600) @[Connor1996](https://github.com/Connor1996) + - When TiKV detects the existence of corrupted SST files, it logs the specific reasons for the corruption [#16308](https://github.com/tikv/tikv/issues/16308) @[overvenus](https://github.com/overvenus) + - Remove unnecessary async blocks to reduce memory usage [#16540](https://github.com/tikv/tikv/issues/16540) @[overvenus](https://github.com/overvenus) + - Accelerate the shutdown speed of TiKV [#16680](https://github.com/tikv/tikv/issues/16680) @[LykxSassinator](https://github.com/LykxSassinator) + ++ PD + + - Upgrade the etcd version to v3.4.30 [#7904](https://github.com/tikv/pd/issues/7904) @[JmPotato](https://github.com/JmPotato) + - Add the monitoring metric for the maximum Request Unit (RU) per second [#7908](https://github.com/tikv/pd/issues/7908) @[nolouch](https://github.com/nolouch) + ++ TiFlash + + - Mitigate the issue that TiFlash might panic due to updating certificates after TLS is enabled [#8535](https://github.com/pingcap/tiflash/issues/8535) @[windtalker](https://github.com/windtalker) + ++ Tools + + + Backup & Restore (BR) + + - BR cleans up empty SST files during data recovery [#16005](https://github.com/tikv/tikv/issues/16005) @[Leavrth](https://github.com/Leavrth) + - Add PITR integration test cases to cover compatibility testing for log backup and adding index acceleration [#51987](https://github.com/pingcap/tidb/issues/51987) @[Leavrth](https://github.com/Leavrth) + - Enhance the tolerance of log backup to merge operations. When encountering a reasonably long merge operation, log backup tasks are less likely to enter the error state [#16554](https://github.com/tikv/tikv/issues/16554) @[YuJuncen](https://github.com/YuJuncen) + - Improve the table creation performance of the `RESTORE` statement in scenarios with large datasets [#48301](https://github.com/pingcap/tidb/issues/48301) @[Leavrth](https://github.com/Leavrth) + - Support pre-allocating Table ID during the restore process to maximize the reuse of Table ID and improve restore performance [#51736](https://github.com/pingcap/tidb/issues/51736) @[Leavrth](https://github.com/Leavrth) + - Remove the invalid verification for active DDL jobs when log backup starts [#52733](https://github.com/pingcap/tidb/issues/52733) @[Leavrth](https://github.com/Leavrth) + - Remove an outdated compatibility check when using Google Cloud Storage (GCS) as the external storage [#50533](https://github.com/pingcap/tidb/issues/50533) @[lance6716](https://github.com/lance6716) + - Increase the number of retries for failures caused by DNS errors [#53029](https://github.com/pingcap/tidb/issues/53029) @[YuJuncen](https://github.com/YuJuncen) + + + TiCDC + + - Improve memory stability during data recovery using redo logs to reduce the probability of OOM [#10900](https://github.com/pingcap/tiflow/issues/10900) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Significantly improve the stability of data replication in transaction conflict scenarios, with up to 10 times performance improvement [#10896](https://github.com/pingcap/tiflow/issues/10896) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Enable the PD client forwarding function to make TiCDC more stable during network isolation between TiCDC and the PD leader [#10849](https://github.com/pingcap/tiflow/issues/10849) @[asddongmen](https://github.com/asddongmen) + - Improve the initialization speed of replication tasks [#11124](https://github.com/pingcap/tiflow/issues/11124) @[asddongmen](https://github.com/asddongmen) + - Initialize replication tasks asynchronously to reduce initialization time for the processor and owner [#10845](https://github.com/pingcap/tiflow/issues/10845) @[sdojjy](https://github.com/sdojjy) + - Detect the Kafka cluster version automatically to improve compatibility with Kafka [#10852](https://github.com/pingcap/tiflow/issues/10852) @[wk989898](https://github.com/wk989898) + +## Bug fixes + ++ TiDB + + - Fix the issue of inconsistent data indexes caused by concurrent DML operations when adding a unique index [#52914](https://github.com/pingcap/tidb/issues/52914) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue of inconsistent data indexes caused by adding indexes with multi-schema changes on partitioned tables [#52080](https://github.com/pingcap/tidb/issues/52080) @[tangenta](https://github.com/tangenta) + - Fix the issue of inconsistent data indexes caused by adding multi-valued indexes [#51162](https://github.com/pingcap/tidb/issues/51162) @[ywqzzy](https://github.com/ywqzzy) + - Fix the issue that DDL operations get stuck due to network problems [#47060](https://github.com/pingcap/tidb/issues/47060) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that TiDB might report an error due to GC when loading statistics at startup [#53592](https://github.com/pingcap/tidb/issues/53592) @[you06](https://github.com/you06) + - Fix the issue that TiDB might send requests to unready TiKV nodes [#50758](https://github.com/pingcap/tidb/issues/50758) @[zyguan](https://github.com/zyguan) + - Fix the issue that Stale Read might miss after a TiKV rolling restart [#52193](https://github.com/pingcap/tidb/issues/52193) @[zyguan](https://github.com/zyguan) + - Fix the issue that data race might occur during KV request retries, leading to TiDB panics [#51921](https://github.com/pingcap/tidb/issues/51921) @[zyguan](https://github.com/zyguan) + - Fix the issue that TiDB might panic when parsing index data [#47115](https://github.com/pingcap/tidb/issues/47115) @[zyguan](https://github.com/zyguan) + - Fix the issue that TiDB might panic when the JOIN condition contains an implicit type conversion [#46556](https://github.com/pingcap/tidb/issues/46556) @[qw4990](https://github.com/qw4990) + - Fix the issue that comparing a column of `YEAR` type with an unsigned integer that is out of range causes incorrect results [#50235](https://github.com/pingcap/tidb/issues/50235) @[qw4990](https://github.com/qw4990) + - Fix the issue that subqueries in an `UPDATE` list might cause TiDB to panic [#52687](https://github.com/pingcap/tidb/issues/52687) @[winoros](https://github.com/winoros) + - Fix the overflow issue of the `Longlong` type in predicates [#45783](https://github.com/pingcap/tidb/issues/45783) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that `SELECT INTO OUTFILE` does not work when clustered indexes are used as predicates [#42093](https://github.com/pingcap/tidb/issues/42093) @[qw4990](https://github.com/qw4990) + - Fix the issue that the TopN operator might be pushed down incorrectly [#37986](https://github.com/pingcap/tidb/issues/37986) @[qw4990](https://github.com/qw4990) + - Fix the issue that an empty projection causes TiDB to panic [#49109](https://github.com/pingcap/tidb/issues/49109) @[winoros](https://github.com/winoros) + - Fix the issue that Index Merge incorrectly pushes partial limit down when index plans are kept ordered [#52947](https://github.com/pingcap/tidb/issues/52947) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that using a view does not work in recursive CTE [#49721](https://github.com/pingcap/tidb/issues/49721) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that unstable unique IDs of columns might cause the `UPDATE` statement to return errors [#53236](https://github.com/pingcap/tidb/issues/53236) @[winoros](https://github.com/winoros) + - Fix the issue that TiDB panics when executing the `SHOW ERRORS` statement with a predicate that is always `true` [#46962](https://github.com/pingcap/tidb/issues/46962) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that the `final` AggMode and the `non-final` AggMode cannot coexist in Massively Parallel Processing (MPP) [#51362](https://github.com/pingcap/tidb/issues/51362) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that a wrong TableDual plan causes empty query results [#50051](https://github.com/pingcap/tidb/issues/50051) @[onlyacat](https://github.com/onlyacat) + - Fix the issue that TiDB might panic when initializing statistics after enabling both `lite-init-stats` and `concurrently-init-stats` [#52223](https://github.com/pingcap/tidb/issues/52223) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that `NO_JOIN` hints do not work with `CREATE BINDING` [#52813](https://github.com/pingcap/tidb/issues/52813) @[qw4990](https://github.com/qw4990) + - Fix the issue that subqueries included in the `ALL` function might cause incorrect results [#52755](https://github.com/pingcap/tidb/issues/52755) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that `VAR_SAMP()` cannot be used as a window function [#52933](https://github.com/pingcap/tidb/issues/52933) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that column pruning without using shallow copies of slices might cause TiDB to panic [#52768](https://github.com/pingcap/tidb/issues/52768) @[winoros](https://github.com/winoros) + - Fix the issue that adding a unique index might cause TiDB to panic [#52312](https://github.com/pingcap/tidb/issues/52312) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that the TiDB server is marked as health before the initialization is complete [#51596](https://github.com/pingcap/tidb/issues/51596) @[shenqidebaozi](https://github.com/shenqidebaozi) + - Fix the issue that the type returned by the `IFNULL` function is inconsistent with MySQL [#51765](https://github.com/pingcap/tidb/issues/51765) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that parallel `Apply` might generate incorrect results when the table has a clustered index [#51372](https://github.com/pingcap/tidb/issues/51372) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that query results might be incorrect when the `HAVING` clause in a subquery contains correlated columns [#51107](https://github.com/pingcap/tidb/issues/51107) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that querying the `TIDB_HOT_REGIONS` table might incorrectly return `INFORMATION_SCHEMA` tables [#50810](https://github.com/pingcap/tidb/issues/50810) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that automatic statistics collection is triggered before the initialization of statistics finishes [#52346](https://github.com/pingcap/tidb/issues/52346) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that AutoID Leader change might cause the value of the auto-increment column to decrease in the case of `AUTO_ID_CACHE=1` [#52600](https://github.com/pingcap/tidb/issues/52600) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that query results might be incorrect when you use Common Table Expressions (CTE) to access partitioned tables with missing statistics [#51873](https://github.com/pingcap/tidb/issues/51873) @[qw4990](https://github.com/qw4990) + - Fix the incorrect calculation and display of the number of connections (Connection Count) on the TiDB Dashboard Monitoring page [#51889](https://github.com/pingcap/tidb/issues/51889) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that DDL operations get stuck when restoring a table with the foreign key [#51838](https://github.com/pingcap/tidb/issues/51838) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that getting the default value of a column returns an error if the column default value is dropped [#50043](https://github.com/pingcap/tidb/issues/50043) [#51324](https://github.com/pingcap/tidb/issues/51324) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that TiDB does not listen to the corresponding port when `force-init-stats` is configured [#51473](https://github.com/pingcap/tidb/issues/51473) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the query result is incorrect when the `IN()` predicate contains `NULL` [#51560](https://github.com/pingcap/tidb/issues/51560) @[winoros](https://github.com/winoros) + - Fix the issue that the TiDB synchronously loading statistics mechanism retries to load empty statistics indefinitely and prints the `fail to get stats version for this histogram` log [#52657](https://github.com/pingcap/tidb/issues/52657) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that `EXCHANGE PARTITION` incorrectly processes foreign keys [#51807](https://github.com/pingcap/tidb/issues/51807) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that `LIMIT` might not be pushed down to the `OR` type `Index Merge` [#48588](https://github.com/pingcap/tidb/issues/48588) @[AilinKid](https://github.com/AilinKid) + - Fix the incorrect result of the TopN operator in correlated subqueries [#52777](https://github.com/pingcap/tidb/issues/52777) @[yibin87](https://github.com/yibin87) + - Fix the issue that the `CPS by type` metric displays incorrect values [#52605](https://github.com/pingcap/tidb/issues/52605) @[nolouch](https://github.com/nolouch) + - Fix the issue that the `EXPLAIN` statement might display incorrect column IDs in the result when statistics for certain columns are not fully loaded [#52207](https://github.com/pingcap/tidb/issues/52207) @[time-and-fate](https://github.com/time-and-fate) + - Fix the issue that expressions containing different collations might cause the query to panic when the new framework for collations is disabled [#52772](https://github.com/pingcap/tidb/issues/52772) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that executing SQL statements containing tables with multi-valued indexes might return the `Can't find a proper physical plan for this query` error [#49438](https://github.com/pingcap/tidb/issues/49438) @[qw4990](https://github.com/qw4990) + - Fix the issue that TiDB cannot correctly convert the type of a system variable in an expression [#43527](https://github.com/pingcap/tidb/issues/43527) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that executing `INSERT IGNORE` might result in inconsistency between the unique index and the data [#51784](https://github.com/pingcap/tidb/issues/51784) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that automatic statistics collection gets stuck after an OOM error occurs [#51993](https://github.com/pingcap/tidb/issues/51993) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that TiDB might crash when `tidb_mem_quota_analyze` is enabled and the memory used by updating statistics exceeds the limit [#52601](https://github.com/pingcap/tidb/issues/52601) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that `max_execute_time` settings at multiple levels interfere with each other [#50914](https://github.com/pingcap/tidb/issues/50914) @[jiyfhust](https://github.com/jiyfhust) + - Fix the issue of index inconsistency caused by adding multiple indexes using a single SQL statement [#51746](https://github.com/pingcap/tidb/issues/51746) @[tangenta](https://github.com/tangenta) + - Fix the issue that the Window function might panic when there is a related subquery in it [#42734](https://github.com/pingcap/tidb/issues/42734) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that TiDB crashes when `shuffleExec` quits unexpectedly [#48230](https://github.com/pingcap/tidb/issues/48230) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that the status gets stuck when rolling back the partition DDL tasks [#51090](https://github.com/pingcap/tidb/issues/51090) @[jiyfhust](https://github.com/jiyfhust) + - Fix the issue that querying JSON of `BINARY` type might cause an error in some cases [#51547](https://github.com/pingcap/tidb/issues/51547) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that adding indexes to large tables fails after enabling the Distributed eXecution Framework (DXF) [#52640](https://github.com/pingcap/tidb/issues/52640) @[tangenta](https://github.com/tangenta) + - Fix the issue that the TTL feature causes data hotspots due to incorrect data range splitting in some cases [#51527](https://github.com/pingcap/tidb/issues/51527) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that `ALTER TABLE ... COMPACT TIFLASH REPLICA` might incorrectly end when the primary key type is `VARCHAR` [#51810](https://github.com/pingcap/tidb/issues/51810) @[breezewish](https://github.com/breezewish) + - Fix the issue of inconsistent data indexes caused by cluster upgrade during adding indexes [#52411](https://github.com/pingcap/tidb/issues/52411) @[tangenta](https://github.com/tangenta) + - Fix the performance regression issue caused by disabling predicate pushdown in TableDual [#50614](https://github.com/pingcap/tidb/issues/50614) @[time-and-fate](https://github.com/time-and-fate) + - Fix the issue that the TiDB server adds a label via the HTTP interface and returns success, but does not take effect [#51427](https://github.com/pingcap/tidb/issues/51427) @[you06](https://github.com/you06) + - Fix the issue that adding indexes in the ingest mode might cause inconsistent data index in some corner cases [#51954](https://github.com/pingcap/tidb/issues/51954) @[lance6716](https://github.com/lance6716) + - Fix the issue that the `init-stats` process might cause TiDB to panic and the `load stats` process to quit [#51581](https://github.com/pingcap/tidb/issues/51581) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the configuration file does not take effect when it contains an invalid configuration item [#51399](https://github.com/pingcap/tidb/issues/51399) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that query execution using MPP might lead to incorrect query results when a SQL statement contains `JOIN` and the `SELECT` list in the statement contains only constants [#50358](https://github.com/pingcap/tidb/issues/50358) @[yibin87](https://github.com/yibin87) + - Fix the issue that in `determinate` mode (`tidb_opt_objective='determinate'`), if a query does not contain predicates, statistics might not be loaded [#48257](https://github.com/pingcap/tidb/issues/48257) @[time-and-fate](https://github.com/time-and-fate) + - Fix the issue that the `SURVIVAL_PREFERENCES` attribute might not appear in the output of the `SHOW CREATE PLACEMENT POLICY` statement under certain conditions [#51699](https://github.com/pingcap/tidb/issues/51699) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that IndexJoin produces duplicate rows when calculating hash values in the Left Outer Anti Semi type [#52902](https://github.com/pingcap/tidb/issues/52902) @[yibin87](https://github.com/yibin87) + - Fix the issue that the `TIMESTAMPADD()` function returns incorrect results [#41052](https://github.com/pingcap/tidb/issues/41052) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that data conversion from the `FLOAT` type to the `UNSIGNED` type returns incorrect results [#41736](https://github.com/pingcap/tidb/issues/41736) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that the `TRUNCATE()` function returns incorrect results when its second argument is a large negative number [#52978](https://github.com/pingcap/tidb/issues/52978) @[yibin87](https://github.com/yibin87) + - Fix the issue that duplicated panel IDs in Grafana might cause abnormal display [#51556](https://github.com/pingcap/tidb/issues/51556) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that TiDB unexpectedly restarts when logging gRPC errors [#51301](https://github.com/pingcap/tidb/issues/51301) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that TiDB loading statistics during startup might cause OOM [#52219](https://github.com/pingcap/tidb/issues/52219) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the TTL job for a table does not stop after the table is deleted [#51540](https://github.com/pingcap/tidb/issues/51540) @[YangKeao](https://github.com/YangKeao) + ++ TiKV + + - Fix the issue that `thread_id` values are incorrectly displayed as `0x5` in TiKV logs [#16398](https://github.com/tikv/tikv/issues/16398) @[overvenus](https://github.com/overvenus) + - Fix the issue of unstable test cases, ensuring that each test uses an independent temporary directory to avoid online configuration changes affecting other test cases [#16871](https://github.com/tikv/tikv/issues/16871) @[glorv](https://github.com/glorv) + - Fix the issue that TiKV might panic during the conversion from Binary to JSON [#16616](https://github.com/tikv/tikv/issues/16616) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that the output of the `raft region` command in tikv-ctl does not include the Region status information [#17037](https://github.com/tikv/tikv/issues/17037) @[glorv](https://github.com/glorv) + - Fix the issue that slow `check-leader` operations on one TiKV node cause `resolved-ts` on other TiKV nodes to fail to advance normally [#15999](https://github.com/tikv/tikv/issues/15999) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that after the process of destroying Peer is interrupted by applying snapshots, it does not resume even after applying snapshots is complete [#16561](https://github.com/tikv/tikv/issues/16561) @[tonyxuqqi](https://github.com/tonyxuqqi) + - Fix the issue that the decimal part of the `DECIMAL` type is incorrect in some cases [#16913](https://github.com/tikv/tikv/issues/16913) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that the `CONV()` function in queries might overflow during numeric system conversion, leading to TiKV panic [#16969](https://github.com/tikv/tikv/issues/16969) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that TiKV converts the time zone incorrectly for Brazil and Egypt [#16220](https://github.com/tikv/tikv/issues/16220) @[overvenus](https://github.com/overvenus) + - Fix the issue that the monitoring metric `tikv_unified_read_pool_thread_count` has no data in some cases [#16629](https://github.com/tikv/tikv/issues/16629) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that inactive Write Ahead Logs (WALs) in RocksDB might corrupt data [#16705](https://github.com/tikv/tikv/issues/16705) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that resolve-ts is blocked when a stale Region peer ignores the GC message [#16504](https://github.com/tikv/tikv/issues/16504) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that during the execution of an optimistic transaction, if other transactions initiate the resolving lock operation on it, there is a small chance that the atomicity of the transaction might be broken if the transaction's primary key has data that was previously committed in Async Commit or 1PC mode [#16620](https://github.com/tikv/tikv/issues/16620) @[MyonKeminta](https://github.com/MyonKeminta) + ++ PD + + - Fix the issue of connection panic after fault recovery in the TiDB network partition [#7926](https://github.com/tikv/pd/issues/7926) @[CabinfeverB](https://github.com/CabinfeverB) + - Fix the issue that scheduling might be incorrectly paused after online data recovery [#8095](https://github.com/tikv/pd/issues/8095) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that the CPS By Type monitoring types display incorrectly after enabling resource groups [#52605](https://github.com/pingcap/tidb/issues/52605) @[nolouch](https://github.com/nolouch) + - Fix the issue that changing the log level via the configuration file does not take effect [#8117](https://github.com/tikv/pd/issues/8117) @[rleungx](https://github.com/rleungx) + - Fix the issue that a large number of retries occur when canceling resource groups queries [#8217](https://github.com/tikv/pd/issues/8217) @[nolouch](https://github.com/nolouch) + - Fix the issue that `ALTER PLACEMENT POLICY` cannot modify the placement policy [#52257](https://github.com/pingcap/tidb/issues/52257) [#51712](https://github.com/pingcap/tidb/issues/51712) @[jiyfhust](https://github.com/jiyfhust) + - Fix the issue that down peers might not recover when using Placement Rules [#7808](https://github.com/tikv/pd/issues/7808) @[rleungx](https://github.com/rleungx) + - Fix the issue that manually transferring the PD leader might fail [#8225](https://github.com/tikv/pd/issues/8225) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that the scheduling of write hotspots might break placement policy constraints [#7848](https://github.com/tikv/pd/issues/7848) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that slots are not fully deleted in a resource group client, which causes the number of the allocated tokens to be less than the specified value [#7346](https://github.com/tikv/pd/issues/7346) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that the scaling progress is not correctly displayed [#7726](https://github.com/tikv/pd/issues/7726) @[CabinfeverB](https://github.com/CabinfeverB) + - Fix the issue that the Leader fails to transfer when you switch it between two deployed data centers [#7992](https://github.com/tikv/pd/issues/7992) @[TonsnakeLin](https://github.com/TonsnakeLin) + - Fix the issue that the `Filter target` monitoring metric for PD does not provide scatter range information [#8125](https://github.com/tikv/pd/issues/8125) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that the query result of `SHOW CONFIG` includes the deprecated configuration item `trace-region-flow` [#7917](https://github.com/tikv/pd/issues/7917) @[rleungx](https://github.com/rleungx) + ++ TiFlash + + - Fix the issue that in the disaggregated storage and compute architecture, null values might be incorrectly returned in queries after adding non-null columns in DDL operations [#9084](https://github.com/pingcap/tiflash/issues/9084) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue of query timeout when executing queries on partitioned tables that contains the empty partition [#9024](https://github.com/pingcap/tiflash/issues/9024) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that in the disaggregated storage and compute architecture, TiFlash might panic when the compute node process is stopped [#8860](https://github.com/pingcap/tiflash/issues/8860) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that querying generated columns returns an error [#8787](https://github.com/pingcap/tiflash/issues/8787) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that TiFlash metadata might become corrupted and cause the process to panic when upgrading a cluster from a version earlier than v6.5.0 to v6.5.0 or later [#9039](https://github.com/pingcap/tiflash/issues/9039) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that the `ENUM` column might cause TiFlash to crash during chunk encoding [#8674](https://github.com/pingcap/tiflash/issues/8674) @[yibin87](https://github.com/yibin87) + - Fix incorrect `local_region_num` values in logs [#8895](https://github.com/pingcap/tiflash/issues/8895) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that in the disaggregated storage and compute architecture, TiFlash might panic during shutdown [#8837](https://github.com/pingcap/tiflash/issues/8837) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash might return transiently incorrect results in high-concurrency read scenarios [#8845](https://github.com/pingcap/tiflash/issues/8845) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that in the disaggregated storage and compute architecture, the disk `used_size` metric displayed in Grafana is incorrect after you modify the value of the `storage.remote.cache.capacity` configuration item for TiFlash compute nodes [#8920](https://github.com/pingcap/tiflash/issues/8920) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that in the disaggregated storage and compute architecture, queries might be permanently blocked after network isolation [#8806](https://github.com/pingcap/tiflash/issues/8806) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that TiFlash might panic when you insert data to columns with invalid default values in non-strict `sql_mode` [#8803](https://github.com/pingcap/tiflash/issues/8803) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + ++ Tools + + + Backup & Restore (BR) + + - Fix a rare issue that special event timing might cause the data loss in log backup [#16739](https://github.com/tikv/tikv/issues/16739) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that too many logs are printed when a full backup fails [#51572](https://github.com/pingcap/tidb/issues/51572) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that a PD connection failure could cause the TiDB instance where the log backup advancer owner is located to panic [#52597](https://github.com/pingcap/tidb/issues/52597) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that the global checkpoint of log backup is advanced ahead of the actual backup file write point due to TiKV restart, which might cause a small amount of backup data loss [#16809](https://github.com/tikv/tikv/issues/16809) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that TiKV might panic when resuming a paused log backup task with unstable network connections to PD [#17020](https://github.com/tikv/tikv/issues/17020) @[YuJuncen](https://github.com/YuJuncen) + - Fix an unstable test case [#52547](https://github.com/pingcap/tidb/issues/52547) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the Region fetched from PD does not have a Leader when restoring data using BR or importing data using TiDB Lightning in physical import mode [#51124](https://github.com/pingcap/tidb/issues/51124) [#50501](https://github.com/pingcap/tidb/issues/50501) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that TiKV panics when a full backup fails to find a peer in some extreme cases [#16394](https://github.com/tikv/tikv/issues/16394) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that removing a log backup task after it is paused does not immediately restore the GC safepoint [#52082](https://github.com/pingcap/tidb/issues/52082) @[3pointer](https://github.com/3pointer) + - Fix the unstable test case `TestClearCache` [#50743](https://github.com/pingcap/tidb/issues/50743) @[3pointer](https://github.com/3pointer) + - Fix the issue that BR fails to restore a transactional KV cluster due to an empty `EndKey` [#52574](https://github.com/pingcap/tidb/issues/52574) @[3pointer](https://github.com/3pointer) + - Fix the issue that the transfer of PD leaders might cause BR to panic when restoring data [#53724](https://github.com/pingcap/tidb/issues/53724) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that BR could not back up the `AUTO_RANDOM` ID allocation progress in a union clustered index that contains an `AUTO_RANDOM` column [#52255](https://github.com/pingcap/tidb/issues/52255) @[Leavrth](https://github.com/Leavrth) + + + TiCDC + + - Fix the issue that high latency in the PD disk I/O causes severe latency in data replication [#9054](https://github.com/pingcap/tiflow/issues/9054) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that calling the API (`/api/v2/owner/resign`) that evicts the TiCDC owner node causes the TiCDC task to restart unexpectedly [#10781](https://github.com/pingcap/tiflow/issues/10781) @[sdojjy](https://github.com/sdojjy) + - Fix the issue that data is written to a wrong CSV file due to wrong BarrierTS in scenarios where DDL statements are executed frequently [#10668](https://github.com/pingcap/tiflow/issues/10668) @[lidezhu](https://github.com/lidezhu) + - Fix the issue that TiCDC fails to validate `TIMESTAMP` type checksum due to time zone mismatch after data integrity validation for single-row data is enabled [#10573](https://github.com/pingcap/tiflow/issues/10573) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that a changefeed with eventual consistency enabled might fail when the object storage sink encounters a temporary failure [#10710](https://github.com/pingcap/tiflow/issues/10710) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that `DROP PRIMARY KEY` and `DROP UNIQUE KEY` statements are not replicated correctly [#10890](https://github.com/pingcap/tiflow/issues/10890) @[asddongmen](https://github.com/asddongmen) + - Fix the issue TiCDC panics when scheduling table replication tasks [#10613](https://github.com/pingcap/tiflow/issues/10613) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that when the downstream Pulsar is stopped, removing the changefeed causes the normal TiCDC process to get stuck, which causes other changefeed processes to get stuck [#10629](https://github.com/pingcap/tiflow/issues/10629) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that restarting PD might cause the TiCDC node to restart with an error [#10799](https://github.com/pingcap/tiflow/issues/10799) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that the old value part of `open-protocol` incorrectly outputs the default value according to the `STRING` type instead of its actual type [#10803](https://github.com/pingcap/tiflow/issues/10803) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that the default value of `TIMEZONE` type is not set according to the correct time zone [#10931](https://github.com/pingcap/tiflow/issues/10931) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that TiCDC fails to execute the `Exchange Partition ... With Validation` DDL downstream after it is written upstream, causing the changefeed to get stuck [#10859](https://github.com/pingcap/tiflow/issues/10859) @[hongyunyan](https://github.com/hongyunyan) + - Fix the issue that data race in the KV client causes TiCDC to panic [#10718](https://github.com/pingcap/tiflow/issues/10718) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that updating the primary key or the unique key in upstream might cause data inconsistency between upstream and downstream [#10918](https://github.com/pingcap/tiflow/issues/10918) @[lidezhu](https://github.com/lidezhu) + + + TiDB Data Migration (DM) + + - Fix the connection blocking issue by upgrading `go-mysql` [#11041](https://github.com/pingcap/tiflow/issues/11041) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that data is lost when the upstream primary key is of binary type [#10672](https://github.com/pingcap/tiflow/issues/10672) @[GMHDBJD](https://github.com/GMHDBJD) + + + TiDB Lightning + + - Fix the issue that TiDB Lightning might fail to import data when EBS BR is running [#49517](https://github.com/pingcap/tidb/issues/49517) @[mittalrishabh](https://github.com/mittalrishabh) + - Fix the issue that TiDB Lightning reports `no database selected` during data import due to incompatible SQL statements in the source files [#51800](https://github.com/pingcap/tidb/issues/51800) @[lance6716](https://github.com/lance6716) + - Fix the issue that killing the PD Leader causes TiDB Lightning to report the `invalid store ID 0` error during data import [#50501](https://github.com/pingcap/tidb/issues/50501) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that TiDB Lightning panics when importing an empty table of Parquet format [#52518](https://github.com/pingcap/tidb/issues/52518) @[kennytm](https://github.com/kennytm) diff --git a/releases/release-7.5.3.md b/releases/release-7.5.3.md new file mode 100644 index 0000000000000..808019b811f3d --- /dev/null +++ b/releases/release-7.5.3.md @@ -0,0 +1,134 @@ +--- +title: TiDB 7.5.3 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 7.5.3. +--- + +# TiDB 7.5.3 Release Notes + +Release date: August 5, 2024 + +TiDB version: 7.5.3 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.5/production-deployment-using-tiup) + +## Compatibility changes + +- Add a new system table [`INFORMATION_SCHEMA.KEYWORDS`](/information-schema/information-schema-keywords.md) to display the information of all keywords supported by TiDB [#48801](https://github.com/pingcap/tidb/issues/48801) @[dveeden](https://github.com/dveeden) +- Change the scope of the TiKV configuration item [`server.grpc-compression-type`](/tikv-configuration-file.md#grpc-compression-type): + + - In v7.5.x versions earlier than v7.5.3, this configuration item only affects the compression algorithm of gRPC messages between TiKV nodes. + - Starting from v7.5.3, this configuration item also affects the compression algorithm of gRPC response messages sent from TiKV to TiDB. Enabling compression might consume more CPU resources. [#17176](https://github.com/tikv/tikv/issues/17176) @[ekexium](https://github.com/ekexium) + +## Improvements + ++ TiDB + + - By batch deleting TiFlash placement rules, improve the processing speed of data GC after performing the `TRUNCATE` or `DROP` operation on partitioned tables [#54068](https://github.com/pingcap/tidb/issues/54068) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + ++ TiFlash + + - Mitigate the issue that TiFlash might panic due to updating certificates after TLS is enabled [#8535](https://github.com/pingcap/tiflash/issues/8535) @[windtalker](https://github.com/windtalker) + - Reduce lock conflicts under highly concurrent data read operations and optimize short query performance [#9125](https://github.com/pingcap/tiflash/issues/9125) @[JinheLin](https://github.com/JinheLin) + ++ Tools + + + Backup & Restore (BR) + + - Except for the `br log restore` subcommand, all other `br log` subcommands support skipping the loading of the TiDB `domain` data structure to reduce memory consumption [#52088](https://github.com/pingcap/tidb/issues/52088) @[Leavrth](https://github.com/Leavrth) + - Support automatically abandoning log backup tasks when encountering a large checkpoint lag, to avoid prolonged blocking GC and potential cluster issues [#50803](https://github.com/pingcap/tidb/issues/50803) @[RidRisR](https://github.com/RidRisR) + - Increase the number of retries for failures caused by DNS errors [#53029](https://github.com/pingcap/tidb/issues/53029) @[YuJuncen](https://github.com/YuJuncen) + - Add PITR integration test cases to cover compatibility testing for log backup and adding index acceleration [#51987](https://github.com/pingcap/tidb/issues/51987) @[Leavrth](https://github.com/Leavrth) + - Increase the number of retries for failures caused by the absence of a leader in a Region [#54017](https://github.com/pingcap/tidb/issues/54017) @[Leavrth](https://github.com/Leavrth) + - Support setting Alibaba Cloud access credentials through environment variables [#45551](https://github.com/pingcap/tidb/issues/45551) @[RidRisR](https://github.com/RidRisR) + + + TiCDC + + - Support directly outputting raw events when the downstream is a Message Queue (MQ) or cloud storage [#11211](https://github.com/pingcap/tiflow/issues/11211) @[CharlesCheung96](https://github.com/CharlesCheung96) + +## Bug fixes + ++ TiDB + + - Fix the issue that loading index statistics might cause memory leaks [#54022](https://github.com/pingcap/tidb/issues/54022) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that the `UPDATE` operation can cause TiDB OOM in multi-table scenarios [#53742](https://github.com/pingcap/tidb/issues/53742) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that indirect placeholder `?` references in a `GROUP BY` statement cannot find columns [#53872](https://github.com/pingcap/tidb/issues/53872) @[qw4990](https://github.com/qw4990) + - Fix the issue that the `LENGTH()` condition is unexpectedly removed when the collation is `utf8_bin` or `utf8mb4_bin` [#53730](https://github.com/pingcap/tidb/issues/53730) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that inserting an overlarge number in scientific notation returns a warning instead of an error, making it consistent with MySQL [#47787](https://github.com/pingcap/tidb/issues/47787) @[qw4990](https://github.com/qw4990) + - Fix the issue that recursive CTE queries might result in invalid pointers [#54449](https://github.com/pingcap/tidb/issues/54449) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that statistics collection does not update the `stats_history` table when encountering duplicate primary keys [#47539](https://github.com/pingcap/tidb/issues/47539) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that when queries contain non-correlated subqueries and `LIMIT` clauses, column pruning might be incomplete, resulting in a less optimal plan [#54213](https://github.com/pingcap/tidb/issues/54213) @[qw4990](https://github.com/qw4990) + - Fix the issue of abnormally high memory usage caused by `memTracker` not being detached when the `HashJoin` or `IndexLookUp` operator is the driven side sub-node of the `Apply` operator [#54005](https://github.com/pingcap/tidb/issues/54005) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that the recursive CTE operator incorrectly tracks memory usage [#54181](https://github.com/pingcap/tidb/issues/54181) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that the memory used by transactions might be tracked multiple times [#53984](https://github.com/pingcap/tidb/issues/53984) @[ekexium](https://github.com/ekexium) + - Fix the issue that using `SHOW WARNINGS;` to obtain warnings might cause a panic [#48756](https://github.com/pingcap/tidb/issues/48756) @[xhebox](https://github.com/xhebox) + - Fix the issue that updating an `UNSIGNED` type of field to `-1` returns `null` instead of `0` when `sql_mode=''` [#47816](https://github.com/pingcap/tidb/issues/47816) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that the `TIMESTAMPADD()` function goes into an infinite loop when the first argument is `month` and the second argument is negative [#54908](https://github.com/pingcap/tidb/issues/54908) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that the Connection Count monitoring metric in Grafana is incorrect when some connections exit before the handshake is complete [#54428](https://github.com/pingcap/tidb/issues/54428) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that the Connection Count of each resource group is incorrect when using TiProxy and resource groups [#54545](https://github.com/pingcap/tidb/issues/54545) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that executing `CREATE OR REPLACE VIEW` concurrently might result in the `table doesn't exist` error [#53673](https://github.com/pingcap/tidb/issues/53673) @[tangenta](https://github.com/tangenta) + - Fix the issue that TiDB might return incorrect query results when you query tables with virtual columns in transactions that involve data modification operations [#53951](https://github.com/pingcap/tidb/issues/53951) @[qw4990](https://github.com/qw4990) + - Fix the issue that executing the `SELECT DISTINCT CAST(col AS DECIMAL), CAST(col AS SIGNED) FROM ...` query might return incorrect results [#53726](https://github.com/pingcap/tidb/issues/53726) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue of incorrect WARNINGS information when using Optimizer Hints [#53767](https://github.com/pingcap/tidb/issues/53767) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the illegal column type `DECIMAL(0,0)` can be created in some cases [#53779](https://github.com/pingcap/tidb/issues/53779) @[tangenta](https://github.com/tangenta) + - Fix the issue that the `memory_quota` hint might not work in subqueries [#53834](https://github.com/pingcap/tidb/issues/53834) @[qw4990](https://github.com/qw4990) + - Fix the issue that JSON-related functions return errors inconsistent with MySQL in some cases [#53799](https://github.com/pingcap/tidb/issues/53799) @[dveeden](https://github.com/dveeden) + - Fix the issue that improper use of metadata locks might lead to writing anomalous data when using the plan cache under certain circumstances [#53634](https://github.com/pingcap/tidb/issues/53634) @[zimulala](https://github.com/zimulala) + - Fix the issue that certain filter conditions in queries might cause the planner module to report an `invalid memory address or nil pointer dereference` error [#53582](https://github.com/pingcap/tidb/issues/53582) [#53580](https://github.com/pingcap/tidb/issues/53580) [#53594](https://github.com/pingcap/tidb/issues/53594) [#53603](https://github.com/pingcap/tidb/issues/53603) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that after a statement within a transaction is killed by OOM, if TiDB continues to execute the next statement within the same transaction, you might get an error `Trying to start aggressive locking while it's already started` and a panic occurs [#53540](https://github.com/pingcap/tidb/issues/53540) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that executing `ALTER TABLE ... REMOVE PARTITIONING` might cause data loss [#53385](https://github.com/pingcap/tidb/issues/53385) @[mjonss](https://github.com/mjonss) + - Fix the issue that `PREPARE`/`EXECUTE` statements with the `CONV` expression containing a `?` argument might result in incorrect query results when executed multiple times [#53505](https://github.com/pingcap/tidb/issues/53505) @[qw4990](https://github.com/qw4990) + - Fix the issue that TiDB fails to reject unauthenticated user connections in some cases when using the `auth_socket` authentication plugin [#54031](https://github.com/pingcap/tidb/issues/54031) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that the query latency of stale reads increases, caused by information schema cache misses [#53428](https://github.com/pingcap/tidb/issues/53428) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that the `STATE` field in the `INFORMATION_SCHEMA.TIDB_TRX` table is empty due to the `size` of the `STATE` field not being defined [#53026](https://github.com/pingcap/tidb/issues/53026) @[cfzjywxk](https://github.com/cfzjywxk) + - Fix the issue that the `tidb_enable_async_merge_global_stats` and `tidb_analyze_partition_concurrency` system variables do not take effect during automatic statistics collection [#53972](https://github.com/pingcap/tidb/issues/53972) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that using `CURRENT_DATE()` as the default value for a column results in incorrect query results [#53746](https://github.com/pingcap/tidb/issues/53746) @[tangenta](https://github.com/tangenta) + - Fix the issue of reusing wrong point get plans for `SELECT ... FOR UPDATE` [#54652](https://github.com/pingcap/tidb/issues/54652) @[qw4990](https://github.com/qw4990) + ++ TiKV + + - Fix the issue that setting the gRPC message compression method via `grpc-compression-type` does not take effect on messages sent from TiKV to TiDB [#17176](https://github.com/tikv/tikv/issues/17176) @[ekexium](https://github.com/ekexium) + - Fix the issue that highly concurrent Coprocessor requests might cause TiKV OOM [#16653](https://github.com/tikv/tikv/issues/16653) @[overvenus](https://github.com/overvenus) + - Fix the issue that CDC and log-backup do not limit the timeout of `check_leader` using the `advance-ts-interval` configuration, causing the `resolved_ts` lag to be too large when TiKV restarts normally in some cases [#17107](https://github.com/tikv/tikv/issues/17107) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that TiKV might repeatedly panic when applying a corrupted Raft data snapshot [#15292](https://github.com/tikv/tikv/issues/15292) @[LykxSassinator](https://github.com/LykxSassinator) + ++ PD + + - Fix the issue that slots are not fully deleted in a resource group client, which causes the number of the allocated tokens to be less than the specified value [#7346](https://github.com/tikv/pd/issues/7346) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that a resource group encounters quota limits when requesting tokens for more than 500 ms [#8349](https://github.com/tikv/pd/issues/8349) @[nolouch](https://github.com/nolouch) + - Fix the data race issue of resource groups [#8267](https://github.com/tikv/pd/issues/8267) @[HuSharp](https://github.com/HuSharp) + - Fix the data race issue that PD encounters during operator checks [#8263](https://github.com/tikv/pd/issues/8263) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that deleted nodes still appear in the candidate connection list in etcd client [#8286](https://github.com/tikv/pd/issues/8286) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that setting the TiKV configuration item [`coprocessor.region-split-size`](/tikv-configuration-file.md#region-split-size) to a value less than 1 MiB causes PD panic [#8323](https://github.com/tikv/pd/issues/8323) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that the encryption manager is not initialized before use [#8384](https://github.com/tikv/pd/issues/8384) @[releungx](https://github.com/releungx) + - Fix the issue that PD logs are not fully redacted when the PD configuration item [`security.redact-info-log`](/pd-configuration-file.md#redact-info-log-new-in-v50) is enabled [#8419](https://github.com/tikv/pd/issues/8419) @[releungx](https://github.com/releungx) + - Fix the issue that no error is reported when binding a role to a resource group [#54417](https://github.com/pingcap/tidb/issues/54417) @[JmPotato](https://github.com/JmPotato) + ++ TiFlash + + - Fix the issue that a large number of duplicate rows might be read in FastScan mode after importing data via BR or TiDB Lightning [#9118](https://github.com/pingcap/tiflash/issues/9118) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that the `SUBSTRING_INDEX()` function might cause TiFlash to crash in some corner cases [#9116](https://github.com/pingcap/tiflash/issues/9116) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that setting the SSL certificate configuration to an empty string in TiFlash incorrectly enables TLS and causes TiFlash to fail to start [#9235](https://github.com/pingcap/tiflash/issues/9235) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that some queries might report a column type mismatch error after late materialization is enabled [#9175](https://github.com/pingcap/tiflash/issues/9175) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that queries with virtual generated columns might return incorrect results after late materialization is enabled [#9188](https://github.com/pingcap/tiflash/issues/9188) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that TiFlash might panic after executing `RENAME TABLE ... TO ...` on a partitioned table with empty partitions across databases [#9132](https://github.com/pingcap/tiflash/issues/9132) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash might panic when a database is deleted shortly after creation [#9266](https://github.com/pingcap/tiflash/issues/9266) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that log backup might be paused after the advancer owner migration [#53561](https://github.com/pingcap/tidb/issues/53561) @[RidRisR](https://github.com/RidRisR) + - Fix the issue that BR fails to correctly identify errors due to multiple nested retries during the restore process [#54053](https://github.com/pingcap/tidb/issues/54053) @[RidRisR](https://github.com/RidRisR) + - Fix the inefficiency issue in scanning DDL jobs during incremental backups [#54139](https://github.com/pingcap/tidb/issues/54139) @[3pointer](https://github.com/3pointer) + - Fix the issue that the backup performance during checkpoint backups is affected due to interruptions in seeking Region leaders [#17168](https://github.com/tikv/tikv/issues/17168) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that after pausing, stopping, and rebuilding the log backup task, the task status is normal, but the checkpoint does not advance [#53047](https://github.com/pingcap/tidb/issues/53047) @[RidRisR](https://github.com/RidRisR) + - Fix the issue that DDLs requiring backfilling, such as `ADD INDEX` and `MODIFY COLUMN`, might not be correctly recovered during incremental restore [#54426](https://github.com/pingcap/tidb/issues/54426) @[3pointer](https://github.com/3pointer) + + + TiCDC + + - Fix the issue that the checksum is not correctly set to `0` after splitting `UPDATE` events [#11402](https://github.com/pingcap/tiflow/issues/11402) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that the Processor module might get stuck when the downstream Kafka is inaccessible [#11340](https://github.com/pingcap/tiflow/issues/11340) @[asddongmen](https://github.com/asddongmen) + + + Dumpling + + - Fix the issue that Dumpling reports an error when exporting tables and views at the same time [#53682](https://github.com/pingcap/tidb/issues/53682) @[tangenta](https://github.com/tangenta) diff --git a/releases/release-7.5.4.md b/releases/release-7.5.4.md new file mode 100644 index 0000000000000..5abb7291674d6 --- /dev/null +++ b/releases/release-7.5.4.md @@ -0,0 +1,125 @@ +--- +title: TiDB 7.5.4 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 7.5.4. +--- + +# TiDB 7.5.4 Release Notes + +Release date: October 15, 2024 + +TiDB version: 7.5.4 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.5/production-deployment-using-tiup) + +## Compatibility changes + +- Set a default limit of 2048 for DDL historical tasks retrieved through the [TiDB HTTP API](https://github.com/pingcap/tidb/blob/release-7.5/docs/tidb_http_api.md) to prevent OOM issues caused by excessive historical tasks [#55711](https://github.com/pingcap/tidb/issues/55711) @[joccau](https://github.com/joccau) + +## Improvements + ++ TiDB + + - Support applying the `tidb_redact_log` setting to the output of `EXPLAIN` statements and further optimize the logic in processing logs [#54565](https://github.com/pingcap/tidb/issues/54565) @[hawkingrei](https://github.com/hawkingrei) + - Optimize the query speed of TiDB slow queries [#54630](https://github.com/pingcap/tidb/pull/54630) @[yibin87](https://github.com/yibin87) + ++ TiKV + + - Optimize the trigger mechanism of RocksDB compaction to accelerate disk space reclamation when handling a large number of DELETE versions [#17269](https://github.com/tikv/tikv/issues/17269) @[AndreMouche](https://github.com/AndreMouche) + - Reduce the memory usage of the peer message channel [#16229](https://github.com/tikv/tikv/issues/16229) @[Connor1996](https://github.com/Connor1996) + - Optimize the jittery access delay when restarting TiKV due to waiting for the log to be applied, improving the stability of TiKV [#15874](https://github.com/tikv/tikv/issues/15874) @[LykxSassinator](https://github.com/LykxSassinator) + - Optimize TiKV's `DiskFull` detection to make it compatible with RaftEngine's `spill-dir` configuration, ensuring that this feature works consistently [#17356](https://github.com/tikv/tikv/issues/17356) @[LykxSassinator](https://github.com/LykxSassinator) + ++ TiFlash + + - Optimize the execution efficiency of `LENGTH()` and `ASCII()` functions [#9344](https://github.com/pingcap/tiflash/issues/9344) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Mitigate the issue that TiFlash might panic due to updating certificates after TLS is enabled [#8535](https://github.com/pingcap/tiflash/issues/8535) @[windtalker](https://github.com/windtalker) + - Improve the cancel mechanism of the JOIN operator, so that the JOIN operator can respond to cancel requests in a timely manner [#9430](https://github.com/pingcap/tiflash/issues/9430) @[windtalker](https://github.com/windtalker) + ++ Tools + + + Backup & Restore (BR) + + - Support checking whether the disk space in TiKV is sufficient before TiKV downloads each SST file. If the space is insufficient, BR terminates the restore and returns an error [#17224](https://github.com/tikv/tikv/issues/17224) @[RidRisR](https://github.com/RidRisR) + + + TiCDC + + - When the downstream is TiDB with the `SUPER` permission granted, TiCDC supports querying the execution status of `ADD INDEX DDL` from the downstream database to avoid data replication failure due to timeout in retrying executing the DDL statement in some cases [#10682](https://github.com/pingcap/tiflow/issues/10682) @[CharlesCheung96](https://github.com/CharlesCheung96) + +## Bug fixes + ++ TiDB + + - Fix the issue that `FLASHBACK DATABASE` fails when many tables exist in the database [#54415](https://github.com/pingcap/tidb/issues/54415) @[lance6716](https://github.com/lance6716) + - Fix the issue that RANGE partitioned tables that are not strictly self-incrementing can be created [#54829](https://github.com/pingcap/tidb/issues/54829) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that a query statement that contains `UNION` might return incorrect results [#52985](https://github.com/pingcap/tidb/issues/52985) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that `INDEX_HASH_JOIN` cannot exit properly when SQL is abnormally interrupted [#54688](https://github.com/pingcap/tidb/issues/54688) @[wshwsh12](https://github.com/wshwsh12) + - Reset the parameters in the `Open` method of `PipelinedWindow` to fix the unexpected error that occurs when the `PipelinedWindow` is used as a child node of `Apply` due to the reuse of previous parameter values caused by repeated opening and closing operations [#53600](https://github.com/pingcap/tidb/issues/53600) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that the query latency of stale reads increases, caused by information schema cache misses [#53428](https://github.com/pingcap/tidb/issues/53428) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that disk files might not be deleted after the `Sort` operator spills and a query error occurs [#55061](https://github.com/pingcap/tidb/issues/55061) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that the query might return incorrect results instead of an error after being killed [#50089](https://github.com/pingcap/tidb/issues/50089) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that table replication fails when the index length of the table replicated from DM exceeds the maximum length specified by `max-index-length` [#55138](https://github.com/pingcap/tidb/issues/55138) @[lance6716](https://github.com/lance6716) + - Fix the issue that the `SUB_PART` value in the `INFORMATION_SCHEMA.STATISTICS` table is `NULL` [#55812](https://github.com/pingcap/tidb/issues/55812) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that an error occurs when a DML statement contains nested generated columns [#53967](https://github.com/pingcap/tidb/issues/53967) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that the `tot_col_size` column in the `mysql.stats_histograms` table might be a negative number [#55126](https://github.com/pingcap/tidb/issues/55126) @[qw4990](https://github.com/qw4990) + - Fix the data race issue in `IndexNestedLoopHashJoin` [#49692](https://github.com/pingcap/tidb/issues/49692) @[solotzg](https://github.com/solotzg) + - Fix the issue that the query might get stuck when terminated because the memory usage exceeds the limit set by `tidb_mem_quota_query` [#55042](https://github.com/pingcap/tidb/issues/55042) @[yibin87](https://github.com/yibin87) + - Fix the issue that `columnEvaluator` cannot identify the column references in the input chunk, which leads to `runtime error: index out of range` when executing SQL statements [#53713](https://github.com/pingcap/tidb/issues/53713) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that the performance of the `SELECT ... WHERE ... ORDER BY ...` statement execution is poor in some cases [#54969](https://github.com/pingcap/tidb/issues/54969) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that empty `groupOffset` in `StreamAggExec` might cause TiDB to panic [#53867](https://github.com/pingcap/tidb/issues/53867) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that TiDB queries cannot be canceled during cop task construction [#55957](https://github.com/pingcap/tidb/issues/55957) @[yibin87](https://github.com/yibin87) + - Fix the issue that an `out of range` error might occur when a small display width is specified for a column of the integer type [#55837](https://github.com/pingcap/tidb/issues/55837) @[windtalker](https://github.com/windtalker) + - Fix the issue that `duplicate entry` might occur when adding unique indexes [#56161](https://github.com/pingcap/tidb/issues/56161) @[tangenta](https://github.com/tangenta) + - Fix the issue that TiDB panics when importing temporary tables using the `IMPORT INTO` statement [#55970](https://github.com/pingcap/tidb/issues/55970) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue of data index inconsistency caused by retries during index addition [#55808](https://github.com/pingcap/tidb/issues/55808) @[lance6716](https://github.com/lance6716) + ++ TiKV + + - Fix the issue that TiKV might panic when a stale replica processes Raft snapshots, triggered by a slow split operation and immediate removal of the new replica [#17469](https://github.com/tikv/tikv/issues/17469) @[hbisheng](https://github.com/hbisheng) + - Fix the flow control issue that might occur after deleting large tables or partitions [#17304](https://github.com/tikv/tikv/issues/17304) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that bloom filters are incompatible between earlier versions (earlier than v7.1) and later versions [#17272](https://github.com/tikv/tikv/issues/17272) @[v01dstar](https://github.com/v01dstar) + - Fix the issue that prevents master key rotation when the master key is stored in a Key Management Service (KMS) [#17410](https://github.com/tikv/tikv/issues/17410) @[hhwyt](https://github.com/hhwyt) + - Fix the issue that the **Storage async write duration** monitoring metric on the TiKV panel in Grafana is inaccurate [#17579](https://github.com/tikv/tikv/issues/17579) @[overvenus](https://github.com/overvenus) + - Fix the issue that when a large number of transactions are queuing for lock release on the same key and the key is frequently updated, excessive pressure on deadlock detection might cause TiKV OOM issues [#17394](https://github.com/tikv/tikv/issues/17394) @[MyonKeminta](https://github.com/MyonKeminta) + ++ PD + + - Fix the issue that PD's Region API cannot be requested when a large number of Regions exist [#55872](https://github.com/pingcap/tidb/issues/55872) @[rleungx](https://github.com/rleungx) + - Fix the issue that when using a wrong parameter in `evict-leader-scheduler`, PD does not report errors correctly and some schedulers are unavailable [#8619](https://github.com/tikv/pd/issues/8619) @[rleungx](https://github.com/rleungx) + - Fix the issue that data race might occur in the scheduling server when a PD leader is switched in the microservice mode [#8538](https://github.com/tikv/pd/issues/8538) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that slots are not fully deleted in a resource group client, which causes the number of the allocated tokens to be less than the specified value [#7346](https://github.com/tikv/pd/issues/7346) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that the time data type in the `INFORMATION_SCHEMA.RUNAWAY_WATCHES` table is incorrect [#54770](https://github.com/pingcap/tidb/issues/54770) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that setting `replication.strictly-match-label` to `true` causes TiFlash to fail to start [#8480](https://github.com/tikv/pd/issues/8480) @[rleungx](https://github.com/rleungx) + ++ TiFlash + + - Fix the issue that TiFlash write nodes might fail to restart in the disaggregated storage and compute architecture [#9282](https://github.com/pingcap/tiflash/issues/9282) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that a network partition (network disconnection) between TiFlash and any PD might cause read request timeout errors [#9243](https://github.com/pingcap/tiflash/issues/9243) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that when using the `CAST()` function to convert a string to a datetime with a time zone or invalid characters, the result is incorrect [#8754](https://github.com/pingcap/tiflash/issues/8754) @[solotzg](https://github.com/solotzg) + - Fix the issue that read snapshots of TiFlash write nodes are not released in a timely manner in the disaggregated storage and compute architecture [#9298](https://github.com/pingcap/tiflash/issues/9298) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that TiFlash fails to parse the table schema when the table contains Bit-type columns with a default value that contains invalid characters [#9461](https://github.com/pingcap/tiflash/issues/9461) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that some queries might report errors when late materialization is enabled [#9472](https://github.com/pingcap/tiflash/issues/9472) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that converting a data type to the `DECIMAL` type might lead to wrong query results in some extreme cases [#53892](https://github.com/pingcap/tidb/issues/53892) @[guo-shaoge](https://github.com/guo-shaoge) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that backup tasks might get stuck if TiKV becomes unresponsive during the backup process [#53480](https://github.com/pingcap/tidb/issues/53480) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the checkpoint path of backup and restore is incompatible with some external storage [#55265](https://github.com/pingcap/tidb/issues/55265) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that after a log backup PITR task fails and you stop it, the safepoints related to that task are not properly cleared in PD [#17316](https://github.com/tikv/tikv/issues/17316) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that BR logs might print sensitive credential information when log backup is enabled [#55273](https://github.com/pingcap/tidb/issues/55273) @[RidRisR](https://github.com/RidRisR) + - Fix the issue that BR integration test cases are unstable, and add a new test case to simulate snapshot or log backup file corruption [#53835](https://github.com/pingcap/tidb/issues/53835) @[Leavrth](https://github.com/Leavrth) + + + TiCDC + + - Fix the issue that the **barrier-ts** monitoring metric for changefeed checkpoints might be inaccurate [#11553](https://github.com/pingcap/tiflow/issues/11553) @[3AceShowHand](https://github.com/3AceShowHand) + + + TiDB Data Migration (DM) + + - Fix the issue that data replication is interrupted when the index length exceeds the default value of `max-index-length` [#11459](https://github.com/pingcap/tiflow/issues/11459) @[michaelmdeng](https://github.com/michaelmdeng) + - Fix the issue that DM does not set the default database when processing the `ALTER DATABASE` statement, which causes a replication error [#11503](https://github.com/pingcap/tiflow/issues/11503) @[lance6716](https://github.com/lance6716) + - Fix the issue that multiple DM-master nodes might simultaneously become leaders, leading to data inconsistency [#11602](https://github.com/pingcap/tiflow/issues/11602) @[GMHDBJD](https://github.com/GMHDBJD) + + + TiDB Lightning + + - Fix the issue that transaction conflicts occur during data import using TiDB Lightning [#49826](https://github.com/pingcap/tidb/issues/49826) @[lance6716](https://github.com/lance6716) diff --git a/releases/release-7.5.5.md b/releases/release-7.5.5.md new file mode 100644 index 0000000000000..d372ac42bd61c --- /dev/null +++ b/releases/release-7.5.5.md @@ -0,0 +1,167 @@ +--- +title: TiDB 7.5.5 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 7.5.5. +--- + +# TiDB 7.5.5 Release Notes + +Release date: December 31, 2024 + +TiDB version: 7.5.5 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.5/production-deployment-using-tiup) + +## Compatibility changes + +- Add a new system variable [`tidb_ddl_reorg_max_write_speed`](https://docs.pingcap.com/tidb/v7.5/system-variables#tidb_ddl_reorg_max_write_speed-new-in-v6512-and-v755) to limit the maximum speed of the ingest phase when adding indexes [#57156](https://github.com/pingcap/tidb/issues/57156) @[CbcWestwolf](https://github.com/CbcWestwolf) +- Change the default value of the TiKV configuration item [`raft-client-queue-size`](/tikv-configuration-file.md#raft-client-queue-size) from `8192` to `16384` [#17101](https://github.com/tikv/tikv/issues/17101) @[Connor1996](https://github.com/Connor1996) + +## Improvements + ++ TiDB + + - Adjust estimation results from 0 to 1 for equality conditions that do not hit TopN when statistics are entirely composed of TopN and the modified row count in the corresponding table statistics is non-zero [#47400](https://github.com/pingcap/tidb/issues/47400) @[terry1purcell](https://github.com/terry1purcell) + ++ TiKV + + - Fix the issue that when Raft and RocksDB are deployed on different disks, the slow disk detection does not work for the disk where RocksDB is located [#17884](https://github.com/tikv/tikv/issues/17884) @[LykxSassinator](https://github.com/LykxSassinator) + - Add slow logs for peer and store messages [#16600](https://github.com/tikv/tikv/issues/16600) @[Connor1996](https://github.com/Connor1996) + - Optimize the jittery access delay when restarting TiKV due to waiting for the log to be applied, improving the stability of TiKV [#15874](https://github.com/tikv/tikv/issues/15874) @[LykxSassinator](https://github.com/LykxSassinator) + ++ TiFlash + + - Mitigate the issue that TiFlash might panic due to updating certificates after TLS is enabled [#8535](https://github.com/pingcap/tiflash/issues/8535) @[windtalker](https://github.com/windtalker) + - Improve the garbage collection speed of outdated data in the background for tables with clustered indexes [#9529](https://github.com/pingcap/tiflash/issues/9529) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Optimize the retry strategy for TiFlash compute nodes in the disaggregated storage and compute architecture to handle exceptions when downloading files from Amazon S3 [#9695](https://github.com/pingcap/tiflash/issues/9695) @[JinheLin](https://github.com/JinheLin) + ++ Tools + + + Backup & Restore (BR) + + - Reduce unnecessary log printing during backup [#55902](https://github.com/pingcap/tidb/issues/55902) @[Leavrth](https://github.com/Leavrth) + + + TiDB Data Migration (DM) + + - Lower the log level for unrecognized MariaDB binlog events [#10204](https://github.com/pingcap/tiflow/issues/10204) @[dveeden](https://github.com/dveeden) + +## Bug fixes + ++ TiDB + + - Fix the issue that TiDB cannot resume Reorg DDL tasks from the previous progress after the DDL owner node is switched [#56506](https://github.com/pingcap/tidb/issues/56506) @[tangenta](https://github.com/tangenta) + - Fix the issue that invalid `NULL` values can be inserted in non-strict mode (`sql_mode = ''`) [#56381](https://github.com/pingcap/tidb/issues/56381) @[joechenrh](https://github.com/joechenrh) + - Fix the issue that the data in the **Stats Healthy Distribution** panel of Grafana might be incorrect [#57176](https://github.com/pingcap/tidb/issues/57176) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue of illegal memory access that might occur when a Common Table Expression (CTE) has multiple data consumers and one consumer exits without reading any data [#55881](https://github.com/pingcap/tidb/issues/55881) @[windtalker](https://github.com/windtalker) + - Fix the issue that existing TTL tasks are executed unexpectedly frequently in a cluster that is upgraded from v6.5 to v7.5 or later [#56539](https://github.com/pingcap/tidb/issues/56539) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that TTL tasks are not canceled after the `tidb_ttl_job_enable` variable is disabled [#57404](https://github.com/pingcap/tidb/issues/57404) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that the query latency of stale reads increases, caused by information schema cache misses [#53428](https://github.com/pingcap/tidb/issues/53428) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that stale read does not strictly verify the timestamp of the read operation, resulting in a small probability of affecting the consistency of the transaction when an offset exists between the TSO and the real physical time [#56809](https://github.com/pingcap/tidb/issues/56809) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that the `AUTO_INCREMENT` field is not correctly set after importing data using the `IMPORT INTO` statement [#56476](https://github.com/pingcap/tidb/issues/56476) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that two DDL Owners might exist at the same time [#54689](https://github.com/pingcap/tidb/issues/54689) @[joccau](https://github.com/joccau) + - Fix the issue that TTL might fail if TiKV is not selected as the storage engine [#56402](https://github.com/pingcap/tidb/issues/56402) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that TiDB does not check the index length limitation when executing `ADD INDEX` [#56930](https://github.com/pingcap/tidb/issues/56930) @[fzzf678](https://github.com/fzzf678) + - Fix the issue that when canceling a TTL task, the corresponding SQL is not killed forcibly [#56511](https://github.com/pingcap/tidb/issues/56511) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that execution plan bindings cannot be created for the multi-table `DELETE` statement with aliases [#56726](https://github.com/pingcap/tidb/issues/56726) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that when using `ANALYZE` to collect statistics for a table, if the table contains expression indexes of virtually generated columns, the execution reports an error [#57079](https://github.com/pingcap/tidb/issues/57079) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that Plan Replayer might report an error when importing a table structure containing Placement Rules [#54961](https://github.com/pingcap/tidb/issues/54961) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that when parsing a database name in CTE, it returns a wrong database name [#54582](https://github.com/pingcap/tidb/issues/54582) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `INSERT ... ON DUPLICATE KEY` statement is not compatible with `mysql_insert_id` [#55965](https://github.com/pingcap/tidb/issues/55965) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that improper use of metadata locks might lead to writing anomalous data when using the plan cache under certain circumstances [#53634](https://github.com/pingcap/tidb/issues/53634) @[zimulala](https://github.com/zimulala) + - Fix the issue that the performance is unstable when adding indexes using Global Sort [#54147](https://github.com/pingcap/tidb/issues/54147) @[tangenta](https://github.com/tangenta) + - Fix the issue that Plan Replayer might report an error when importing a table structure containing foreign keys [#56456](https://github.com/pingcap/tidb/issues/56456) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that using the `RANGE COLUMNS` partition function and the `utf8mb4_0900_ai_ci` collation at the same time could result in incorrect query results [#57261](https://github.com/pingcap/tidb/issues/57261) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that using subqueries after the `NATURAL JOIN` or `USING` clause might result in errors [#53766](https://github.com/pingcap/tidb/issues/53766) @[dash12653](https://github.com/dash12653) + - Fix the issue that TTL tasks cannot be canceled when there is a write conflict [#56422](https://github.com/pingcap/tidb/issues/56422) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that if a CTE contains the `ORDER BY`, `LIMIT`, or `SELECT DISTINCT` clause and is referenced by the recursive part of another CTE, it might be incorrectly inlined and result in an execution error [#56603](https://github.com/pingcap/tidb/issues/56603) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that the `UPDATE` statement incorrectly updates values of the `ENUM` type [#56832](https://github.com/pingcap/tidb/issues/56832) @[xhebox](https://github.com/xhebox) + - Fix the issue that executing `RECOVER TABLE BY JOB JOB_ID;` might cause TiDB to panic [#55113](https://github.com/pingcap/tidb/issues/55113) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that the `read_from_storage` hint might not take effect when the query has an available Index Merge execution plan [#56217](https://github.com/pingcap/tidb/issues/56217) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that `INDEX_HASH_JOIN` might hang during an abnormal exit [#54055](https://github.com/pingcap/tidb/issues/54055) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that querying system tables related to the Distributed eXecution Framework (DXF) might lead to upgrade failures [#49263](https://github.com/pingcap/tidb/issues/49263) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that the DDL internal transaction error `GC life time is shorter than transaction duration` causes index addition to fail [#57043](https://github.com/pingcap/tidb/issues/57043) @[tangenta](https://github.com/tangenta) + - Fix the issue that when executing `EXCHANGE PARTITION` and encountering invalid rows, InfoSchema is fully loaded and the error `failed to load schema diff` is reported [#56685](https://github.com/pingcap/tidb/issues/56685) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that collation is not handled correctly when `tidb_ddl_enable_fast_reorg` and `new_collations_enabled_on_first_bootstrap` are enabled, resulting in inconsistent data indexes [#58036](https://github.com/pingcap/tidb/issues/58036) @[djshow832](https://github.com/djshow832) + - Fix the issue that data indexes are inconsistent because plan cache uses the wrong schema when adding indexes [#56733](https://github.com/pingcap/tidb/issues/56733) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that executing `ALTER TABLE TIFLASH REPLICA` during upgrade causes the TiDB node to crash [#57863](https://github.com/pingcap/tidb/issues/57863) @[tangenta](https://github.com/tangenta) + - Fix the issue that the performance of querying `INFORMATION_SCHEMA.columns` degrades [#58184](https://github.com/pingcap/tidb/issues/58184) @[lance6716](https://github.com/lance6716) + - Fix the issue that the default timeout for querying the TiFlash system table is too short [#57816](https://github.com/pingcap/tidb/issues/57816) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that the value of the `default_collation_for_utf8mb4` variable does not work for the `SET NAMES` statement [#56439](https://github.com/pingcap/tidb/issues/56439) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that TTL internal coroutine might panic when you manually delete a timer in the `mysql.tidb_timer` table [#57112](https://github.com/pingcap/tidb/issues/57112) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that when converting a normal table to a partitioned table using the `ALTER TABLE` statement, insufficient checks might result in incorrect data [#55721](https://github.com/pingcap/tidb/issues/55721) @[mjonss](https://github.com/mjonss) + - Fix the issue that when setting `tidb_gogc_tuner_max_value` and `tidb_gogc_tuner_min_value`, if the maximum value is null, an incorrect warning message occurs [#57889](https://github.com/pingcap/tidb/issues/57889) @[hawkingrei](https://github.com/hawkingrei) + - Fix the potential data race issue that might occur in TiDB's internal coroutine [#57798](https://github.com/pingcap/tidb/issues/57798) [#56053](https://github.com/pingcap/tidb/issues/56053) @[fishiu](https://github.com/fishiu) @[tiancaiamao](https://github.com/tiancaiamao) + - Update `golang-jwt` and `jwt` to prevent potential security risks [#57135](https://github.com/pingcap/tidb/issues/57135) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that concurrent writes might result in duplicate data when converting a table with clustered indexes to a partitioned table using the `ALTER TABLE` statement [#57510](https://github.com/pingcap/tidb/issues/57510) @[mjonss](https://github.com/mjonss) + ++ TiKV + + - Fix the issue that merging Regions might cause TiKV to panic in rare cases [#17840](https://github.com/tikv/tikv/issues/17840) @[glorv](https://github.com/glorv) + - Fix the issue that when Raft and RocksDB are deployed on different disks, the slow disk detection does not work for the disk where RocksDB is located [#17884](https://github.com/tikv/tikv/issues/17884) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that the output of jprof is not correctly captured and processed when the `log-file` parameter is not specified [#17607](https://github.com/tikv/tikv/issues/17607) @[Hexilee](https://github.com/Hexilee) + - Fix the issue that latency might increase when hibernated Regions are awake [#17101](https://github.com/tikv/tikv/issues/17101) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that TiKV might panic when executing queries containing `RADIANS()` or `DEGREES()` functions [#17852](https://github.com/tikv/tikv/issues/17852) @[gengliqi](https://github.com/gengliqi) + - Fix the panic issue that occurs when read threads access outdated indexes in the MemTable of the Raft Engine [#17383](https://github.com/tikv/tikv/issues/17383) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that when a large number of transactions are queuing for lock release on the same key and the key is frequently updated, excessive pressure on deadlock detection might cause TiKV OOM issues [#17394](https://github.com/tikv/tikv/issues/17394) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that write jitter might occur when all hibernated Regions are awakened [#17101](https://github.com/tikv/tikv/issues/17101) @[hhwyt](https://github.com/hhwyt) + - Fix the issue that TiKV converts the time zone incorrectly for Brazil and Egypt [#16220](https://github.com/tikv/tikv/issues/16220) @[overvenus](https://github.com/overvenus) + - Fix the issue that Online Unsafe Recovery cannot handle merge abort [#15580](https://github.com/tikv/tikv/issues/15580) @[v01dstar](https://github.com/v01dstar) + - Fix the issue that CPU profiling flag is not reset correctly when an error occurs [#17234](https://github.com/tikv/tikv/issues/17234) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that large batch writes cause performance jitter when `raft-entry-max-size` is set too high [#17701](https://github.com/tikv/tikv/issues/17701) @[SpadeA-Tang](https://github.com/SpadeA-Tang) + - Fix the issue that improper error handling in the conflict detection interface of the import module might cause TiKV to panic [#17830](https://github.com/tikv/tikv/issues/17830) @[joccau](https://github.com/joccau) + ++ PD + + - Fix the issue that when creating `evict-leader-scheduler` or `grant-leader-scheduler` encounters an error, the error message is not returned to pd-ctl [#8759](https://github.com/tikv/pd/issues/8759) @[okJiang](https://github.com/okJiang) + - Fix the issue that PD cannot quickly re-elect a leader during etcd leader transition [#8823](https://github.com/tikv/pd/issues/8823) @[rleungx](https://github.com/rleungx) + - Fix the memory leak issue in label statistics [#8700](https://github.com/tikv/pd/issues/8700) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that `evict-leader-scheduler` fails to work properly when it is repeatedly created with the same Store ID [#8756](https://github.com/tikv/pd/issues/8756) @[okJiang](https://github.com/okJiang) + - Fix the issue that slots are not fully deleted in a resource group client, which causes the number of the allocated tokens to be less than the specified value [#7346](https://github.com/tikv/pd/issues/7346) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that when using a wrong parameter in `evict-leader-scheduler`, PD does not report errors correctly and some schedulers are unavailable [#8619](https://github.com/tikv/pd/issues/8619) @[rleungx](https://github.com/rleungx) + - Fix the memory leak issue in hotspot cache [#8698](https://github.com/tikv/pd/issues/8698) @[lhy1024](https://github.com/lhy1024) + - Fix the performance jitter issue caused by frequent creation of random number generator [#8674](https://github.com/tikv/pd/issues/8674) @[rleungx](https://github.com/rleungx) + ++ TiFlash + + - Fix the TiFlash panic issue when TiFlash encounters conflicts during concurrent DDL execution [#8578](https://github.com/pingcap/tiflash/issues/8578) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that `LPAD()` and `RPAD()` functions return incorrect results in some cases [#9465](https://github.com/pingcap/tiflash/issues/9465) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that the `SUBSTRING()` function returns incorrect results when the second parameter is negative [#9604](https://github.com/pingcap/tiflash/issues/9604) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that TiFlash fails to parse the table schema when the table contains Bit-type columns with a default value that contains invalid characters [#9461](https://github.com/pingcap/tiflash/issues/9461) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that querying new columns might return incorrect results under the disaggregated storage and compute architecture [#9665](https://github.com/pingcap/tiflash/issues/9665) @[zimulala](https://github.com/zimulala) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that when resuming from a checkpoint after data restore fails, an error `the target cluster is not fresh` occurs [#50232](https://github.com/pingcap/tidb/issues/50232) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that log backups cannot resolve residual locks promptly, causing the checkpoint to fail to advance [#57134](https://github.com/pingcap/tidb/issues/57134) @[3pointer](https://github.com/3pointer) + - Fix the issue that logs might print out encrypted information [#57585](https://github.com/pingcap/tidb/issues/57585) @[kennytm](https://github.com/kennytm) + - Fix the issue that the `TestStoreRemoved` test case is unstable [#52791](https://github.com/pingcap/tidb/issues/52791) @[YuJuncen](https://github.com/YuJuncen) + - Fix potential security vulnerabilities by upgrading the `k8s.io/api` library version [#57790](https://github.com/pingcap/tidb/issues/57790) @[BornChanger](https://github.com/BornChanger) + - Fix the issue that PITR tasks might return the `Information schema is out of date` error when there are a large number of tables in the cluster but the actual data size is small [#57743](https://github.com/pingcap/tidb/issues/57743) @[Tristan1900](https://github.com/Tristan1900) + + + TiCDC + + - Fix the issue that TiCDC might report an error when replicating a `TRUNCATE TABLE` DDL on a table without valid index [#11765](https://github.com/pingcap/tiflow/issues/11765) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that the `tableID` for partitioned tables is not correctly set in Simple Protocol messages [#11846](https://github.com/pingcap/tiflow/issues/11846) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that the redo module fails to properly report errors [#11744](https://github.com/pingcap/tiflow/issues/11744) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that after filtering out `add table partition` events is configured in `ignore-event`, TiCDC does not replicate other types of DML changes for related partitions to the downstream [#10524](https://github.com/pingcap/tiflow/issues/10524) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that TiCDC mistakenly discards DDL tasks when the schema versions of DDL tasks become non-incremental during TiDB DDL owner changes [#11714](https://github.com/pingcap/tiflow/issues/11714) @[wlwilliamx](https://github.com/wlwilliamx) + + + TiDB Data Migration (DM) + + - Fix the issue that automatically generated IDs in a table might have significant jumps after data import in physical import mode [#11768](https://github.com/pingcap/tiflow/issues/11768) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that the pre-check of `start-task` fails when both TLS and `shard-mode` are configured [#11842](https://github.com/pingcap/tiflow/issues/11842) @[sunxiaoguang](https://github.com/sunxiaoguang) + - Fix the issue that connecting to MySQL 8.0 fails when the password length exceeds 19 characters [#11603](https://github.com/pingcap/tiflow/issues/11603) @[fishiu](https://github.com/fishiu) + + + TiDB Lightning + + - Fix the issue that TiDB Lightning fails to receive oversized messages sent from TiKV [#56114](https://github.com/pingcap/tidb/issues/56114) @[fishiu](https://github.com/fishiu) + - Fix the issue that the `AUTO_INCREMENT` value is set too high after importing data using the physical import mode [#56814](https://github.com/pingcap/tidb/issues/56814) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that TiDB Lightning does not automatically retry when encountering `Lock wait timeout` errors during metadata updates [#53042](https://github.com/pingcap/tidb/issues/53042) @[guoshouyan](https://github.com/guoshouyan) + - Fix the issue that the performance degrades when importing data from a cloud storage in high-concurrency scenarios [#57413](https://github.com/pingcap/tidb/issues/57413) @[xuanyu66](https://github.com/xuanyu66) + - Fix the issue that TiDB Lightning might get stuck for a long time during the preparation phase when importing a large number of Parquet files [#56104](https://github.com/pingcap/tidb/issues/56104) @[zeminzhou](https://github.com/zeminzhou) + - Fix the issue that the error report output is truncated when importing data using TiDB Lightning [#58085](https://github.com/pingcap/tidb/issues/58085) @[lance6716](https://github.com/lance6716) + + + Dumpling + + - Fix the issue that Dumpling fails to retry properly when receiving a 503 error from Google Cloud Storage (GCS) [#56127](https://github.com/pingcap/tidb/issues/56127) @[OliverS929](https://github.com/OliverS929) diff --git a/releases/release-7.5.6.md b/releases/release-7.5.6.md new file mode 100644 index 0000000000000..dc43357dff058 --- /dev/null +++ b/releases/release-7.5.6.md @@ -0,0 +1,137 @@ +--- +title: TiDB 7.5.6 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 7.5.6. +--- + +# TiDB 7.5.6 Release Notes + +Release date: March 14, 2025 + +TiDB version: 7.5.6 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.5/production-deployment-using-tiup) + +## Compatibility changes + +- Support the openEuler 22.03 LTS SP3/SP4 operating system. For more information, see [OS and platform requirements](https://docs.pingcap.com/tidb/v7.5/hardware-and-software-requirements#os-and-platform-requirements). + +## Improvements + ++ TiDB + + - Adjust estimation results from 0 to 1 for equality conditions that do not hit TopN when statistics are entirely composed of TopN and the modified row count in the corresponding table statistics is non-zero [#47400](https://github.com/pingcap/tidb/issues/47400) @[terry1purcell](https://github.com/terry1purcell) + - Enhance the timestamp validity check [#57786](https://github.com/pingcap/tidb/issues/57786) @[MyonKeminta](https://github.com/MyonKeminta) + - Limit the execution of GC for TTL tables and related statistics collection tasks to the owner node, thereby reducing overhead [#59357](https://github.com/pingcap/tidb/issues/59357) @[lcwangchao](https://github.com/lcwangchao) + ++ TiKV + + - Add the detection mechanism for invalid `max_ts` updates [#17916](https://github.com/tikv/tikv/issues/17916) @[ekexium](https://github.com/ekexium) + - Add slow logs for peer and store messages [#16600](https://github.com/tikv/tikv/issues/16600) @[Connor1996](https://github.com/Connor1996) + - Optimize the jittery access delay when restarting TiKV due to waiting for the log to be applied, improving the stability of TiKV [#15874](https://github.com/tikv/tikv/issues/15874) @[LykxSassinator](https://github.com/LykxSassinator) + ++ TiFlash + + - Mitigate the issue that TiFlash might panic due to updating certificates after TLS is enabled [#8535](https://github.com/pingcap/tiflash/issues/8535) @[windtalker](https://github.com/windtalker) + ++ Tools + + + Backup & Restore (BR) + + - Disable the table-level checksum calculation during full backups by default (`--checksum=false`) to improve backup performance [#56373](https://github.com/pingcap/tidb/issues/56373) @[Tristan1900](https://github.com/Tristan1900) + - Add a check to verify whether the target cluster contains a table with the same name for non-full restore [#55087](https://github.com/pingcap/tidb/issues/55087) @[RidRisR](https://github.com/RidRisR) + + + TiDB Lightning + + - Add a row width check when parsing CSV files to prevent OOM issues [#58590](https://github.com/pingcap/tidb/issues/58590) @[D3Hunter](https://github.com/D3Hunter) + +## Bug fixes + ++ TiDB + + - Fix the issue that some predicates might be lost when constructing `IndexMerge` [#58476](https://github.com/pingcap/tidb/issues/58476) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `ONLY_FULL_GROUP_BY` setting does not take effect on statements in views [#53175](https://github.com/pingcap/tidb/issues/53175) @[mjonss](https://github.com/mjonss) + - Fix the issue that creating two views with the same name does not report an error [#58769](https://github.com/pingcap/tidb/issues/58769) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that converting data from the `BIT` type to the `CHAR` type might cause TiKV panics [#56494](https://github.com/pingcap/tidb/issues/56494) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that a TTL job that loses the heartbeat blocks other jobs from getting heartbeats [#57915](https://github.com/pingcap/tidb/issues/57915) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that querying partitioned tables using an `IN` condition containing a mismatched value type and a type conversion error leads to incorrect query results [#54746](https://github.com/pingcap/tidb/issues/54746) @[mjonss](https://github.com/mjonss) + - Fix the issue that the default value of the `BIT` column is incorrect [#57301](https://github.com/pingcap/tidb/issues/57301) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that in the Prepare protocol, an error occurs when the client uses a non-UTF8 character set [#58870](https://github.com/pingcap/tidb/issues/58870) @[xhebox](https://github.com/xhebox) + - Fix the issue that using variables or parameters in the `CREATE VIEW` statement does not report errors [#53176](https://github.com/pingcap/tidb/issues/53176) @[mjonss](https://github.com/mjonss) + - Fix the issue that loading statistics manually might fail when the statistics file contains null values [#53966](https://github.com/pingcap/tidb/issues/53966) @[King-Dylan](https://github.com/King-Dylan) + - Fix the issue that when querying the `information_schema.cluster_slow_query` table, if the time filter is not added, only the latest slow log file is queried [#56100](https://github.com/pingcap/tidb/issues/56100) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that querying temporary tables might trigger unexpected TiKV requests in some cases [#58875](https://github.com/pingcap/tidb/issues/58875) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the data in the **Stats Healthy Distribution** panel of Grafana might be incorrect [#57176](https://github.com/pingcap/tidb/issues/57176) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that some TTL jobs might hang when modifying `tidb_ttl_delete_rate_limit` [#58484](https://github.com/pingcap/tidb/issues/58484) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that improper exception handling for statistics causes in-memory statistics to be mistakenly deleted when background tasks time out [#57901](https://github.com/pingcap/tidb/issues/57901) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that using `ORDER BY` when querying `cluster_slow_query table` might generate unordered results [#51723](https://github.com/pingcap/tidb/issues/51723) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that when a virtual generated column's dependencies contain a column with the `ON UPDATE` attribute, the data of the updated row and its index data might be inconsistent [#56829](https://github.com/pingcap/tidb/issues/56829) @[joechenrh](https://github.com/joechenrh) + - Fix the issue that TTL jobs cannot be canceled if the TiDB heartbeat is lost [#57784](https://github.com/pingcap/tidb/issues/57784) @[YangKeao](https://github.com/YangKeao) + - When the parameter is of `Enum`, `Bit`, or `Set` type, the `Conv()` function is no longer pushed down to TiKV [#51877](https://github.com/pingcap/tidb/issues/51877) @[yibin87](https://github.com/yibin87) + - Fix the issue that after executing `ALTER TABLE ... PLACEMENT POLICY ...` in a cluster with TiFlash nodes in the disaggregated storage and compute architecture, Region peers might be accidentally added to TiFlash Compute nodes [#58633](https://github.com/pingcap/tidb/issues/58633) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that job status is overwritten when the DDL owner changes [#52747](https://github.com/pingcap/tidb/issues/52747) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that queries with `is null` conditions on Hash partitioned tables cause a panic [#58374](https://github.com/pingcap/tidb/issues/58374) @[Defined2014](https://github.com/Defined2014/) + - Fix the issue that an error occurs when querying partitioned tables that contain generated columns [#58475](https://github.com/pingcap/tidb/issues/58475) @[joechenrh](https://github.com/joechenrh) + - Fix the issue that TTL jobs might be ignored or processed multiple times [#59347](https://github.com/pingcap/tidb/issues/59347) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that incorrect judgment in exchange partition causes execution failure [#59534](https://github.com/pingcap/tidb/issues/59534) @[mjonss](https://github.com/mjonss) + - Fix the issue that setting the `tidb_audit_log` variable with multi-level relative paths causes errors in the log directory [#58971](https://github.com/pingcap/tidb/issues/58971) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that different data types on both sides of the equality condition in Join might cause incorrect results in TiFlash [#59877](https://github.com/pingcap/tidb/issues/59877) @[yibin87](https://github.com/yibin87) + ++ TiKV + + - Fix the issue that the leader could not be quickly elected after Region split [#17602](https://github.com/tikv/tikv/issues/17602) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that time rollback might cause abnormal RocksDB flow control, leading to performance jitter [#17995](https://github.com/tikv/tikv/issues/17995) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that the latest written data might not be readable when only one-phase commit (1PC) is enabled and Async Commit is not enabled [#18117](https://github.com/tikv/tikv/issues/18117) @[zyguan](https://github.com/zyguan) + - Fix the issue that TiKV converts the time zone incorrectly for Brazil and Egypt [#16220](https://github.com/tikv/tikv/issues/16220) @[overvenus](https://github.com/overvenus) + - Fix the issue that a deadlock might occur when GC Worker is under heavy load [#18214](https://github.com/tikv/tikv/issues/18214) @[zyguan](https://github.com/zyguan) + - Fix the issue that disk stalls might prevent leader migration, leading to performance jitter [#17363](https://github.com/tikv/tikv/issues/17363) @[hhwyt](https://github.com/hhwyt) + - Fix the issue that encoding might fail when processing GBK/GB18030 encoded data [#17618](https://github.com/tikv/tikv/issues/17618) @[CbcWestwolf](https://github.com/CbcWestwolf) + - Fix the issue that CDC connections might cause resource leakage when encountering exceptions [#18245](https://github.com/tikv/tikv/issues/18245) @[wlwilliamx](https://github.com/wlwilliamx) + - Fix the issue that Region merge might lead to TiKV abnormal exit due to Raft index mismatch [#18129](https://github.com/tikv/tikv/issues/18129) @[glorv](https://github.com/glorv) + - Fix the issue that Resolved-TS monitoring and logs might be abnormal [#17989](https://github.com/tikv/tikv/issues/17989) @[ekexium](https://github.com/ekexium) + - Fix the issue that an incompatibility with the Titan component causes upgrade failure [#18263](https://github.com/tikv/tikv/issues/18263) @[v01dstar](https://github.com/v01dstar) @[LykxSassinator](https://github.com/LykxSassinator) + ++ PD + + - Fix the issue that the default value of `max-size` for a single log file is not correctly set [#9037](https://github.com/tikv/pd/issues/9037) @[rleungx](https://github.com/rleungx) + - Fix the issue that the value of the `flow-round-by-digit` configuration item might be overwritten after a restart [#8980](https://github.com/tikv/pd/issues/8980) @[nolouch](https://github.com/nolouch) + - Fix the issue that operations in data import or adding index scenarios might fail due to unstable PD network [#8962](https://github.com/tikv/pd/issues/8962) @[okJiang](https://github.com/okJiang) + - Fix the issue that PD might panic when the `tidb_enable_tso_follower_proxy` system variable is enabled [#8950](https://github.com/tikv/pd/issues/8950) @[okJiang](https://github.com/okJiang) + - Fix the issue that the `tidb_enable_tso_follower_proxy` system variable might not take effect [#8947](https://github.com/tikv/pd/issues/8947) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that memory leaks might occur when allocating TSOs [#9004](https://github.com/tikv/pd/issues/9004) @[rleungx](https://github.com/rleungx) + - Fix the issue that Region syncer might not exit in time during the PD Leader switch [#9017](https://github.com/tikv/pd/issues/9017) @[rleungx](https://github.com/rleungx) + - Fix the issue that a PD node might still generate TSOs even when it is not the Leader [#9051](https://github.com/tikv/pd/issues/9051) @[rleungx](https://github.com/rleungx) + ++ TiFlash + + - Fix the issue that TiFlash might unexpectedly reject processing Raft messages when memory usage is low [#9745](https://github.com/pingcap/tiflash/issues/9745) @[CalvinNeo](https://github.com/CalvinNeo) + - Fix the issue that TiFlash might maintain high memory usage after importing large amounts of data [#9812](https://github.com/pingcap/tiflash/issues/9812) @[CalvinNeo](https://github.com/CalvinNeo) + - Fix the issue that queries on a partitioned table might return errors after executing `ALTER TABLE ... RENAME COLUMN` on that partitioned table [#9787](https://github.com/pingcap/tiflash/issues/9787) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that TiFlash might fail to print error stack traces when it unexpectedly exits in certain situations [#9902](https://github.com/pingcap/tiflash/issues/9902) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash startup might be blocked when `profiles.default.init_thread_count_scale` is set to `0` [#9906](https://github.com/pingcap/tiflash/issues/9906) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that a `Not found column` error might occur when a query involves virtual columns and triggers remote reads [#9561](https://github.com/pingcap/tiflash/issues/9561) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that in the disaggregated storage and compute architecture, TiFlash compute nodes might be incorrectly selected as target nodes for adding Region peers [#9750](https://github.com/pingcap/tiflash/issues/9750) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that log backup fails to exit properly when encountering a fatal error due to not being able to access PD [#18087](https://github.com/tikv/tikv/issues/18087) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that BR fails to restore due to getting the `rpcClient is idle` error when sending requests to TiKV [#58845](https://github.com/pingcap/tidb/issues/58845) @[Tristan1900](https://github.com/Tristan1900) + - Fix the issue that PITR fails to restore indexes larger than 3072 bytes [#58430](https://github.com/pingcap/tidb/issues/58430) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that the `status` field is missing in the result when querying log backup tasks using `br log status --json` [#57959](https://github.com/pingcap/tidb/issues/57959) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that log backup might unexpectedly enter a paused state when the advancer owner switches [#58031](https://github.com/pingcap/tidb/issues/58031) @[3pointer](https://github.com/3pointer) + + + TiCDC + + - Fix the issue that enabling TiCDC in scenarios with many small tables might cause TiKV to restart [#18142](https://github.com/tikv/tikv/issues/18142) @[hicqu](https://github.com/hicqu) + - Fix the issue that after the default value of a newly added column in the upstream is changed from `NOT NULL` to `NULL`, the default values of that column in the downstream are incorrect [#12037](https://github.com/pingcap/tiflow/issues/12037) @[wk989898](https://github.com/wk989898) + - Fix the issue that out-of-order messages resent by the Sarama client cause Kafka message order to be incorrect [#11935](https://github.com/pingcap/tiflow/issues/11935) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that TiCDC uses incorrect table names for filtering during `RENAME TABLE` operations [#11946](https://github.com/pingcap/tiflow/issues/11946) @[wk989898](https://github.com/wk989898) + - Fix the issue that goroutines leak occurs after a changefeed is deleted [#11954](https://github.com/pingcap/tiflow/issues/11954) @[hicqu](https://github.com/hicqu) + - Fix the issue that using the `--overwrite-checkpoint-ts` parameter in the `changefeed pause` command might cause the changefeed to be stuck [#12055](https://github.com/pingcap/tiflow/issues/12055) @[hongyunyan](https://github.com/hongyunyan) + - Fix the issue that TiCDC reports errors when replicating `default NULL` SQL statements via the Avro protocol [#11994](https://github.com/pingcap/tiflow/issues/11994) @[wk989898](https://github.com/wk989898) + - Fix the issue that TiCDC fails to properly connect to PD after PD scale-in [#12004](https://github.com/pingcap/tiflow/issues/12004) @[lidezhu](https://github.com/lidezhu) + + + TiDB Lightning + + - Fix the issue that logs are not properly desensitized [#59086](https://github.com/pingcap/tidb/issues/59086) @[GMHDBJD](https://github.com/GMHDBJD) diff --git a/releases/release-7.5.7.md b/releases/release-7.5.7.md new file mode 100644 index 0000000000000..1846ee9617fb6 --- /dev/null +++ b/releases/release-7.5.7.md @@ -0,0 +1,167 @@ +--- +title: TiDB 7.5.7 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 7.5.7. +--- + +# TiDB 7.5.7 Release Notes + +Release date: September 4, 2025 + +TiDB version: 7.5.7 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.5/production-deployment-using-tiup) + +## Compatibility changes + +- Change the default value of [`tidb_enable_historical_stats`](https://docs.pingcap.com/tidb/v7.5/system-variables/#tidb_enable_historical_stats) from `ON` to `OFF`, which turns off historical statistics to avoid potential stability issues [#53048](https://github.com/pingcap/tidb/issues/53048) @[hawkingrei](https://github.com/hawkingrei) +- TiKV deprecates the following configuration items and replaces them with the new [`gc.auto-compaction`](https://docs.pingcap.com/tidb/v7.5/tikv-configuration-file/#gcauto-compaction) configuration group, which controls automatic compaction behavior [#18727](https://github.com/tikv/tikv/issues/18727) @[v01dstar](https://github.com/v01dstar) + + - Deprecated configuration items: [`region-compact-check-interval`](https://docs.pingcap.com/tidb/v7.5/tikv-configuration-file#region-compact-check-interval), [`region-compact-check-step`](https://docs.pingcap.com/tidb/v7.5/tikv-configuration-file#region-compact-check-step), [`region-compact-min-tombstones`](https://docs.pingcap.com/tidb/v7.5/tikv-configuration-file#region-compact-min-tombstones), [`region-compact-tombstones-percent`](https://docs.pingcap.com/tidb/v7.5/tikv-configuration-file#region-compact-tombstones-percent), [`region-compact-min-redundant-rows`](https://docs.pingcap.com/tidb/v7.5/tikv-configuration-file#region-compact-min-redundant-rows-new-in-v710), and [`region-compact-redundant-rows-percent`](https://docs.pingcap.com/tidb/v7.5/tikv-configuration-file#region-compact-redundant-rows-percent-new-in-v710). + - New configuration items: [`gc.auto-compaction.check-interval`](https://docs.pingcap.com/tidb/v7.5/tikv-configuration-file#check-interval-new-in-v757), [`gc.auto-compaction.tombstone-num-threshold`](https://docs.pingcap.com/tidb/v7.5/tikv-configuration-file#tombstone-num-threshold-new-in-v757), [`gc.auto-compaction.tombstone-percent-threshold`](https://docs.pingcap.com/tidb/v7.5/tikv-configuration-file#tombstone-percent-threshold-new-in-v757), [`gc.auto-compaction.redundant-rows-threshold`](https://docs.pingcap.com/tidb/v7.5/tikv-configuration-file#redundant-rows-threshold-new-in-v757), [`gc.auto-compaction.redundant-rows-percent-threshold`](https://docs.pingcap.com/tidb/v7.5/tikv-configuration-file#redundant-rows-percent-threshold-new-in-v757), and [`gc.auto-compaction.bottommost-level-force`](https://docs.pingcap.com/tidb/v7.5/tikv-configuration-file#bottommost-level-force-new-in-v757). + +## Improvements + ++ TiDB + + - Add flow control interfaces for Region splitting and data ingestion during data import [#61553](https://github.com/pingcap/tidb/issues/61553) @[tangenta](https://github.com/tangenta) + - Optimize the performance of obtaining data distribution information when performing simple queries on tables with large data volumes [#53850](https://github.com/pingcap/tidb/issues/53850) @[you06](https://github.com/you06) + - Add a monitoring metric to observe the write speed to TiKV during index addition [#60925](https://github.com/pingcap/tidb/issues/60925) @[CbcWestwolf](https://github.com/CbcWestwolf) + - Optimize the locking logic of DML during DDL execution to reduce lock conflicts between DML and DDL operations, improving DDL performance in some scenarios. However, due to the additional secondary index locking operations, DML performance might decrease slightly [#62337](https://github.com/pingcap/tidb/issues/62337) @[lcwangchao](https://github.com/lcwangchao) + - Improve the behavior when the system variable [`tidb_opt_ordering_index_selectivity_threshold`](/system-variables.md#tidb_opt_ordering_index_selectivity_threshold-new-in-v700) is set to `1`, enhancing the control capability of this variable [#60242](https://github.com/pingcap/tidb/issues/60242) @[time-and-fate](https://github.com/time-and-fate) + - Avoid refreshing statistics across the entire cluster after `ANALYZE` statement execution, reducing the execution time of `ANALYZE` [#57631](https://github.com/pingcap/tidb/issues/57631) @[0xPoe](https://github.com/0xPoe) + - Support constant folding for columns with `NOT NULL` constraints, folding `IS NULL` evaluations into `FALSE` [#62050](https://github.com/pingcap/tidb/issues/62050) @[hawkingrei](https://github.com/hawkingrei) + - The optimizer supports constant propagation in more types of `JOIN` operations [#51700](https://github.com/pingcap/tidb/issues/51700) @[hawkingrei](https://github.com/hawkingrei) + - Improve the performance of temporary index merging when extensive lock conflicts exist between DML and DDL operations [#61433](https://github.com/pingcap/tidb/issues/61433) @[tangenta](https://github.com/tangenta) + ++ TiKV + + - Optimize the trigger logic of TiKV compaction to process all data segments in order of reclamation efficiency, reducing the performance impact of MVCC redundant data [#18571](https://github.com/tikv/tikv/issues/18571) @[v01dstar](https://github.com/v01dstar) + - Optimize the tail latency of async snapshot and write operations in environments with a large number of SST files [#18743](https://github.com/tikv/tikv/issues/18743) @[Connor1996](https://github.com/Connor1996) + - Improve the speed of Region Merge in scenarios with empty tables and small Regions [#17376](https://github.com/tikv/tikv/issues/17376) @[LykxSassinator](https://github.com/LykxSassinator) + - Optimize the handling of `CompactedEvent` in Raftstore by moving it to the `split-check` worker, reducing blocking on the main Raftstore thread [#18532](https://github.com/tikv/tikv/issues/18532) @[LykxSassinator](https://github.com/LykxSassinator) + - Add metrics for memory usage per thread [#15927](https://github.com/tikv/tikv/issues/15927) @[Connor1996](https://github.com/Connor1996) + - Log only `SST ingest is experiencing slowdowns` when SST ingest is too slow, and skip calling `get_sst_key_ranges` to avoid performance jitter [#18549](https://github.com/tikv/tikv/issues/18549) @[LykxSassinator](https://github.com/LykxSassinator) + - Optimize the jittery access delay when restarting TiKV due to waiting for the log to be applied, improving the stability of TiKV [#15874](https://github.com/tikv/tikv/issues/15874) @[LykxSassinator](https://github.com/LykxSassinator) + - Optimize the cleanup mechanism of residual data to mitigate the impact on request latency [#18107](https://github.com/tikv/tikv/issues/18107) @[LykxSassinator](https://github.com/LykxSassinator) + - Optimize the performance of `fetch_entries_to` in Raft Engine to reduce contention and improve performance under mixed workloads [#18605](https://github.com/tikv/tikv/issues/18605) @[LykxSassinator](https://github.com/LykxSassinator) + - Support dynamically modifying flow-control configurations for write operations [#17395](https://github.com/tikv/tikv/issues/17395) @[glorv](https://github.com/glorv) + - Support ingesting SST files without blocking foreground writes, reducing the impact of latency [#18081](https://github.com/tikv/tikv/issues/18081) @[hhwyt](https://github.com/hhwyt) + - Optimize the detection mechanism for I/O jitter on KvDB disks when KvDB and RaftDB use separate mount paths [#18463](https://github.com/tikv/tikv/issues/18463) @[LykxSassinator](https://github.com/LykxSassinator) + - Add slow logs for peer and store messages [#16600](https://github.com/tikv/tikv/issues/16600) @[Connor1996](https://github.com/Connor1996) + ++ PD + + - Add GO runtime-related monitoring metrics in Prometheus [#8931](https://github.com/tikv/pd/issues/8931) @[bufferflies](https://github.com/bufferflies) + - Reduce unnecessary error logs [#9370](https://github.com/tikv/pd/issues/9370) @[bufferflies](https://github.com/bufferflies) + ++ TiFlash + + - Enhance the observability for TiFlash OOM risks in wide table scenarios [#10272](https://github.com/pingcap/tiflash/issues/10272) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Increase the maximum retry count when acquiring storage snapshots to improve query stability for large tables [#10300](https://github.com/pingcap/tiflash/issues/10300) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - When deploying TiDB on Amazon EC2, BR supports AWS Instance Metadata Service Version 2 (IMDSv2). You can configure your EC2 instance to allow BR to use the IAM role associated with the instance for appropriate permissions to access Amazon S3 [#16443](https://github.com/tikv/tikv/issues/16443) @[pingyu](https://github.com/pingyu) + - The Download API of TiKV supports filtering out data within a certain time range when downloading backup files, which avoids importing outdated or future data versions during restore [#18399](https://github.com/tikv/tikv/issues/18399) @[3pointer](https://github.com/3pointer) + +## Bug fixes + ++ TiDB + + - Fix the issue that shared KV requests in `IndexMerge` and `IndexLookUp` operators cause data races when pushing down queries [#60175](https://github.com/pingcap/tidb/issues/60175) @[you06](https://github.com/you06) + - Fix a potential goroutine leak issue in the Hash Aggregation operator [#58004](https://github.com/pingcap/tidb/issues/58004) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that MPP plans might not be selected when indexes on generated columns are set as visible [#47766](https://github.com/pingcap/tidb/issues/47766) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that SQL statements containing `_charset(xxx), _charset(xxx2), ...` generate different digests [#58447](https://github.com/pingcap/tidb/issues/58447) @[xhebox](https://github.com/xhebox) + - Fix the issue that frequent Region merges prevent TTL jobs from starting [#61512](https://github.com/pingcap/tidb/issues/61512) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that TiFlash query results are inconsistent after executing a lossy DDL statement [#61455](https://github.com/pingcap/tidb/issues/61455) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue of incorrect key range in `ALTER RANGE meta SET PLACEMENT POLICY` [#60888](https://github.com/pingcap/tidb/issues/60888) @[nolouch](https://github.com/nolouch) + - Fix the issue that the data in the **Stats Healthy Distribution** panel of Grafana might be incorrect [#57176](https://github.com/pingcap/tidb/issues/57176) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the comparison behavior of `latin1_bin` differs from that of `utf8mb4_bin` and `utf8_bin` [#60701](https://github.com/pingcap/tidb/issues/60701) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that after disabling metadata locking (MDL), DDL operations get stuck after failing to update the schema version [#61210](https://github.com/pingcap/tidb/issues/61210) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that enabling Redact log does not take effect in certain scenarios [#59279](https://github.com/pingcap/tidb/issues/59279) @[tangenta](https://github.com/tangenta) + - Fix the issue that a TiDB session might crash when Fix Control #44855 is enabled [#59762](https://github.com/pingcap/tidb/issues/59762) @[winoros](https://github.com/winoros) + - Remove redundant log entries when the `IndexLookup` operator encounters a `context canceled` error [#61072](https://github.com/pingcap/tidb/issues/61072) @[yibin87](https://github.com/yibin87) + - Fix the issue that improper exception handling for statistics causes in-memory statistics to be mistakenly deleted when background tasks time out [#57901](https://github.com/pingcap/tidb/issues/57901) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that executing `ADD UNIQUE INDEX` might cause data inconsistency [#60339](https://github.com/pingcap/tidb/issues/60339) @[tangenta](https://github.com/tangenta) + - Fix the issue that non-public indexes are shown in the statistics system table [#60430](https://github.com/pingcap/tidb/issues/60430) @[tangenta](https://github.com/tangenta) + - Fix the issue that the `Close()` method of the Hash Join v1 operator fails to recover from a panic [#60926](https://github.com/pingcap/tidb/issues/60926) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that shallow copy of `PhysicalExchangeSender.HashCol` causes TiFlash to crash or generate incorrect results [#60517](https://github.com/pingcap/tidb/issues/60517) @[windtalker](https://github.com/windtalker) + - Fix the issue that statistics for tables of the `BIT` type cannot be loaded [#62289](https://github.com/pingcap/tidb/issues/62289) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that statistics for `BIT` type columns fail to be loaded into memory [#59759](https://github.com/pingcap/tidb/issues/59759) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that when the disk-spilling operation of an `ANALYZE` statement takes too long in extreme cases, other TiDB nodes might fail to update the latest statistics [#54552](https://github.com/pingcap/tidb/issues/54552) @[0xPoe](https://github.com/0xPoe) + - Fix the issue that when the collected column statistics are all in TopN, the row count estimation might remain 0 even after subsequent writes [#47400](https://github.com/pingcap/tidb/issues/47400) @[terry1purcell](https://github.com/terry1purcell) + - Fix the issue that the estimated cost displayed in `explain format="cost_trace"` might be incorrect [#61155](https://github.com/pingcap/tidb/issues/61155) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the cost formulas displayed in `explain format="cost_trace"` might contain empty parentheses [#61127](https://github.com/pingcap/tidb/issues/61127) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that circular foreign key definitions cause infinite loops [#60985](https://github.com/pingcap/tidb/issues/60985) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that internal queries might fail to construct index range queries properly when using `NULL` [#62196](https://github.com/pingcap/tidb/issues/62196) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that Plan Cache stores an incorrect execution plan, causing execution errors [#56772](https://github.com/pingcap/tidb/issues/56772) @[dash12653](https://github.com/dash12653) + - Fix the issue that row count estimates across months or years can be significantly overestimated [#50080](https://github.com/pingcap/tidb/issues/50080) @[terry1purcell](https://github.com/terry1purcell) + - Fix the issue that the concurrency of `ANALYZE` subtasks greatly exceeds the configured limit [#61785](https://github.com/pingcap/tidb/issues/61785) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that an expression-based TopN sort item is incorrectly generated during TopN pushdown [#60655](https://github.com/pingcap/tidb/issues/60655) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that TiDB might print panic logs in the background when column or index statistics are missing [#61733](https://github.com/pingcap/tidb/issues/61733) @[winoros](https://github.com/winoros) + - Fix the issue that row count estimation for `JOIN` can be highly inaccurate when column or index statistics are missing [#61602](https://github.com/pingcap/tidb/issues/61602) @[qw4990](https://github.com/qw4990) + - Fix the issue that the default value of the system variable `tidb_cost_model_version` is set incorrectly [#61565](https://github.com/pingcap/tidb/issues/61565) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that statistics might be incorrect when the first column of a table is a virtual generated column [#61606](https://github.com/pingcap/tidb/issues/61606) @[winoros](https://github.com/winoros) + - Fix the issue that Plan Cache is incorrectly skipped with predicate simplification [#61513](https://github.com/pingcap/tidb/issues/61513) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that executing `ADMIN CANCEL DDL JOBS` while adding an index causes the index-adding process to hang [#61087](https://github.com/pingcap/tidb/issues/61087) @[tangenta](https://github.com/tangenta) + - Fix the issue that `ADMIN CHECK` still returns success even after some internal SQL executions fail [#61612](https://github.com/pingcap/tidb/issues/61612) @[joechenrh](https://github.com/joechenrh) + - Fix the issue that data and indexes became inconsistent after adding multiple indexes through multi-schema change [#61255](https://github.com/pingcap/tidb/issues/61255) @[tangenta](https://github.com/tangenta) + ++ TiKV + + - Fix the issue that a deadlock might be triggered during CPU profiling [#18474](https://github.com/tikv/tikv/issues/18474) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that Online Unsafe Recovery might be blocked by certain TiFlash replicas, preventing the commit index from advancing [#18197](https://github.com/tikv/tikv/issues/18197) @[v01dstar](https://github.com/v01dstar) + - Fix the issue that TiKV might use a compression algorithm that the client cannot decode [#18079](https://github.com/tikv/tikv/issues/18079) @[ekexium](https://github.com/ekexium) + - Fix the issue that TiKV allows excessive SST ingest requests under high concurrency [#18452](https://github.com/tikv/tikv/issues/18452) @[hbisheng](https://github.com/hbisheng) + - Fix the issue that `Ingestion picked level` and `Compaction Job Size(files)` are displayed incorrectly in the TiKV dashboard in Grafana [#15990](https://github.com/tikv/tikv/issues/15990) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that the unexpected `Server is busy` error occurs after TiKV restarts [#18233](https://github.com/tikv/tikv/issues/18233) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that TiKV converts the time zone incorrectly for Brazil and Egypt [#16220](https://github.com/tikv/tikv/issues/16220) @[overvenus](https://github.com/overvenus) + - Fix misleading descriptions in `StoreMsg` log entries in slow logs [#18561](https://github.com/tikv/tikv/issues/18561) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix incorrect thread memory metrics [#18125](https://github.com/tikv/tikv/issues/18125) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that TiKV fails to terminate ongoing manual compaction tasks during graceful shutdown [#18396](https://github.com/tikv/tikv/issues/18396) @[LykxSassinator](https://github.com/LykxSassinator) + ++ PD + + - Fix the issue that the `split-merge-interval` configuration item might not take effect when you modify its value repeatedly (such as changing it from `1s` to `1h` and back to `1s`) [#8404](https://github.com/tikv/pd/issues/8404) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that the default value of `lease` is not correctly set [#9156](https://github.com/tikv/pd/issues/9156) @[rleungx](https://github.com/rleungx) + - Fix the issue that improperly closing TiDB Dashboard TCP connections could lead to PD goroutine leaks [#9402](https://github.com/tikv/pd/issues/9402) @[baurine](https://github.com/baurine) + - Fix the issue that newly added TiKV nodes might fail to be scheduled [#9145](https://github.com/tikv/pd/issues/9145) @[bufferflies](https://github.com/bufferflies) + - Fix the issue that enabling `tidb_enable_tso_follower_proxy` might cause the TSO service to become unavailable [#9188](https://github.com/tikv/pd/issues/9188) @[Tema](https://github.com/Tema) + ++ TiFlash + + - Fix the panic issue caused by accidental deletion of SST files during running `IMPORT INTO` or `BR restore` [#10141](https://github.com/pingcap/tiflash/issues/10141) @[CalvinNeo](https://github.com/CalvinNeo) + - Fix the issue that creating an expression index in the form of `((NULL))` causes TiFlash to panic [#9891](https://github.com/pingcap/tiflash/issues/9891) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash might panic when handling snapshots with irregular Region key-ranges [#10147](https://github.com/pingcap/tiflash/issues/10147) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash might consume a large amount of memory when tables in a cluster contain a large number of `ENUM` type columns [#9947](https://github.com/pingcap/tiflash/issues/9947) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash might fail to restart after inserting a single row of data larger than 16 MiB [#10052](https://github.com/pingcap/tiflash/issues/10052) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that missing resource control low token signals lead to query throttling [#10137](https://github.com/pingcap/tiflash/issues/10137) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that TiFlash might return the `Exception: Block schema mismatch` error when executing SQL statements containing `GROUP BY ... WITH ROLLUP` [#10110](https://github.com/pingcap/tiflash/issues/10110) @[gengliqi](https://github.com/gengliqi) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that PITR fails to restore indexes larger than 3072 bytes [#58430](https://github.com/pingcap/tidb/issues/58430) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that log backup uploads to Azure Blob Storage are slow when transferring large volumes of data [#18410](https://github.com/tikv/tikv/issues/18410) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that BR does not check whether the corresponding table exists in the cluster when filtering tables with `-f` [#61592](https://github.com/pingcap/tidb/issues/61592) @[RidRisR](https://github.com/RidRisR) + + + TiCDC + + - Fix the issue that the changefeed might get stuck when using external storage as the downstream [#9162](https://github.com/pingcap/tiflow/issues/9162) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that the changefeed might get stuck after the replication traffic exceeds the traffic threshold of the downstream Kafka [#12110](https://github.com/pingcap/tiflow/issues/12110) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that using the `--overwrite-checkpoint-ts` parameter in the `changefeed pause` command might cause the changefeed to be stuck [#12055](https://github.com/pingcap/tiflow/issues/12055) @[hongyunyan](https://github.com/hongyunyan) + - Fix the issue that evaluating event filter expressions on tables containing virtual columns might cause a panic [#12206](https://github.com/pingcap/tiflow/issues/12206) @[lidezhu](https://github.com/lidezhu) + - Fix the issue of case-sensitive matching for column and index names in the dispatcher configuration [#12103](https://github.com/pingcap/tiflow/issues/12103) @[wk989898](https://github.com/wk989898) + - Fix the issue that the resolved ts lag keeps increasing after scaling in or out TiKV nodes on the same IP address because of outdated store IDs [#12162](https://github.com/pingcap/tiflow/issues/12162) @[3AceShowHand](https://github.com/3AceShowHand) + + + TiDB Lightning + + - Fix the issue that TiDB Lightning can get stuck for several hours when importing Parquet files from cloud storage into TiDB [#60224](https://github.com/pingcap/tidb/issues/60224) @[joechenrh](https://github.com/joechenrh) + - Fix the issue that TiDB Lightning returns the `context deadline exceeded` error when the RPC request to TiKV times out [#61326](https://github.com/pingcap/tidb/issues/61326) @[OliverS929](https://github.com/OliverS929) + + + NG Monitoring + + - Fix the issue that TSDB consumes too much memory under high cardinality of time series data, and provide a memory configuration option for TSDB [#295](https://github.com/pingcap/ng-monitoring/issues/295) @[mornyx](https://github.com/mornyx) diff --git a/releases/release-notes.md b/releases/release-notes.md index ebad82bfaaf39..aae9f88b5ff42 100644 --- a/releases/release-notes.md +++ b/releases/release-notes.md @@ -1,10 +1,23 @@ --- title: Release Notes -aliases: ['/docs/dev/releases/release-notes/','/docs/dev/releases/rn/'] +summary: TiDB has released multiple versions, including 7.5.1, 7.5.0, 7.4.0-DMR, 7.3.0-DMR, 7.2.0-DMR, 7.1.4, 7.1.3, 7.1.2, 7.1.1, 7.1.0, 7.0.0-DMR, 6.6.0-DMR, 6.5.9, 6.5.8, 6.5.7, 6.5.6, 6.5.5, 6.5.4, 6.5.3, 6.5.2, 6.5.1, 6.5.0, 6.4.0-DMR, 6.3.0-DMR, 6.2.0-DMR, 6.1.7, 6.1.6, 6.1.5, 6.1.4, 6.1.3, 6.1.2, 6.1.1, 6.1.0, 6.0.0-DMR, 5.4.3, 5.4.2, 5.4.1, 5.4.0, 5.3.4, 5.3.3, 5.3.2, 5.3.1, 5.3.0, 5.2.4, 5.2.3, 5.2.2, 5.2.1, 5.2.0, 5.1.5, 5.1.4, 5.1.3, 5.1.2, 5.1.1, 5.1.0, 5.0.6, 5.0.5, 5.0.4, 5.0.3, 5.0.2, 5.0.1, 5.0.0, 5.0.0-rc, 4.0.16, 4.0.15, 4.0.14, 4.0.13, 4.0.12, 4.0.11, 4.0.10, 4.0.9, 4.0.8, 4.0.7, 4.0.6, 4.0.5, 4.0.4, 4.0.3, 4.0.2, 4.0.1, 4.0.0, 4.0.0-rc.2, 4.0.0-rc.1, 4.0.0-rc, 4.0.0-beta.2, 4.0.0-beta.1, 4.0.0-beta, 3.1.2, 3.1.1, 3.1.0, 3.1.0-rc, 3.1.0-beta.2, 3.1.0-beta.1, 3.1.0-beta, 3.0.20, 3.0.19, 3.0.18, 3.0.17, 3.0.16, 3.0.15, 3.0.14, 3.0.13, 3.0.12, 3.0.11, 3.0.10, 3.0.9, 3.0.8, 3.0.7, 3.0.6, 3.0.5, 3.0.4, 3.0.3, 3.0.2, 3.0.1, 3.0.0, 3.0.0-rc.3, 3.0.0-rc.2, 3.0.0-rc.1, 3.0.0-beta.1, 3.0.0-beta, 2.1.19, 2.1.18, 2.1.17, 2.1.16, 2.1.15, 2.1.14, 2.1.13, 2.1.12, 2.1.11, 2.1.10, 2.1.9, 2.1.8, 2.1.7, 2.1.6, 2.1.5, 2.1.4, 2.1.3, 2.1.2, 2.1.1, 2.1.0, 2.1.0-rc.5, 2.1.0-rc.4, 2.1.0-rc.3, 2.1.0-rc.2, 2.1.0-rc.1, 2.1.0-beta, 2.0.11, 2.0.10, 2.0.9, 2.0.8, 2.0.7, 2.0.6, 2.0.5, 2.0.4, 2.0.3, 2.0.2, 2.0.1, 2.0.0, 2.0.0-rc.5, 2.0.0-rc.4, 2.0.0-rc.3, 2.0.0-rc.1, 1.1.0-beta, 1.1.0-alpha, 1.0.8, 1.0.7, 1.0.6, 1.0.5, 1.0.4, 1.0.3, 1.0.2, 1.0.1, 1.0.0, Pre-GA, rc4, rc3, rc2, rc1. --- # TiDB Release Notes + + +## 7.5 + +- [7.5.7](/releases/release-7.5.7.md): 2025-09-04 +- [7.5.6](/releases/release-7.5.6.md): 2025-03-14 +- [7.5.5](/releases/release-7.5.5.md): 2024-12-31 +- [7.5.4](/releases/release-7.5.4.md): 2024-10-15 +- [7.5.3](/releases/release-7.5.3.md): 2024-08-05 +- [7.5.2](/releases/release-7.5.2.md): 2024-06-13 +- [7.5.1](/releases/release-7.5.1.md): 2024-02-29 +- [7.5.0](/releases/release-7.5.0.md): 2023-12-01 + ## 7.4 - [7.4.0-DMR](/releases/release-7.4.0.md): 2023-10-12 @@ -19,6 +32,10 @@ aliases: ['/docs/dev/releases/release-notes/','/docs/dev/releases/rn/'] ## 7.1 +- [7.1.6](/releases/release-7.1.6.md): 2024-11-21 +- [7.1.5](/releases/release-7.1.5.md): 2024-04-26 +- [7.1.4](/releases/release-7.1.4.md): 2024-03-11 +- [7.1.3](/releases/release-7.1.3.md): 2023-12-21 - [7.1.2](/releases/release-7.1.2.md): 2023-10-25 - [7.1.1](/releases/release-7.1.1.md): 2023-07-24 - [7.1.0](/releases/release-7.1.0.md): 2023-05-31 @@ -33,6 +50,13 @@ aliases: ['/docs/dev/releases/release-notes/','/docs/dev/releases/rn/'] ## 6.5 +- [6.5.12](/releases/release-6.5.12.md): 2025-02-27 +- [6.5.11](/releases/release-6.5.11.md): 2024-09-20 +- [6.5.10](/releases/release-6.5.10.md): 2024-06-20 +- [6.5.9](/releases/release-6.5.9.md): 2024-04-12 +- [6.5.8](/releases/release-6.5.8.md): 2024-02-02 +- [6.5.7](/releases/release-6.5.7.md): 2024-01-08 +- [6.5.6](/releases/release-6.5.6.md): 2023-12-07 - [6.5.5](/releases/release-6.5.5.md): 2023-09-21 - [6.5.4](/releases/release-6.5.4.md): 2023-08-28 - [6.5.3](/releases/release-6.5.3.md): 2023-06-14 diff --git a/releases/release-pre-ga.md b/releases/release-pre-ga.md index effafb4181013..733ae093dc762 100644 --- a/releases/release-pre-ga.md +++ b/releases/release-pre-ga.md @@ -1,6 +1,6 @@ --- title: Pre-GA release notes -aliases: ['/docs/dev/releases/release-pre-ga/','/docs/dev/releases/prega/'] +summary: TiDB Pre-GA release on August 30, 2017, focuses on MySQL compatibility, SQL optimization, stability, and performance. TiDB introduces SQL query optimizer enhancements, MySQL compatibility, JSON type support, and memory consumption reduction. Placement Driver (PD) now supports manual leader change, while TiKV uses dedicated Rocksdb for Raft log storage and improves performance. TiDB Connector for Spark Beta Release implements predicates pushdown, aggregation pushdown, and range pruning, capable of running TPC+H queries. --- # Pre-GA Release Notes diff --git a/releases/release-rc.1.md b/releases/release-rc.1.md index f3f1962ffd8d4..bc29abdfada79 100644 --- a/releases/release-rc.1.md +++ b/releases/release-rc.1.md @@ -1,6 +1,6 @@ --- title: TiDB RC1 Release Notes -aliases: ['/docs/dev/releases/release-rc.1/','/docs/dev/releases/rc1/'] +summary: TiDB RC1 was released on December 23, 2016. Updates include improved write speed and reduced disk space usage in TiKV, optimized scheduling strategy framework in PD, and added features in the SQL query optimizer and new tools in TiDB. The release also supports more built-in functions in MySQL and enhances the speed of the `add index` statement. --- # TiDB RC1 Release Notes @@ -19,7 +19,7 @@ On December 23, 2016, TiDB RC1 is released. See the following updates in this re + The scheduling strategy framework is optimized and now the strategy is more flexible and reasonable. + The support for `label` is added to support Cross Data Center scheduling. -+ PD Controller is provided to operate the PD cluster more easily. ++ PD Control is provided to operate the PD cluster more easily. ## TiDB diff --git a/releases/release-rc.2.md b/releases/release-rc.2.md index 6a62d4d86c678..4ca6cecc1f5c4 100644 --- a/releases/release-rc.2.md +++ b/releases/release-rc.2.md @@ -1,6 +1,6 @@ --- title: TiDB RC2 Release Notes -aliases: ['/docs/dev/releases/release-rc.2/','/docs/dev/releases/rc2/'] +summary: TiDB RC2, released on March 1, 2017, focuses on MySQL compatibility, SQL query optimization, system stability, and performance. It introduces a new permission management mechanism, allowing users to control data access similar to MySQL privilege management. Key improvements include query optimizer enhancements, basic privilege management support, MySQL built-in functions, and performance optimizations. PD now supports location aware replica scheduling and fast scheduling based on region count, while TiKV introduces Async Apply for improved write performance and various optimizations for read and insert performance. Bug fixes and memory leak solutions are also included. --- # TiDB RC2 Release Notes diff --git a/releases/release-rc.3.md b/releases/release-rc.3.md index c63980ff53e5d..b7dc6c5798e70 100644 --- a/releases/release-rc.3.md +++ b/releases/release-rc.3.md @@ -1,6 +1,6 @@ --- title: TiDB RC3 Release Notes -aliases: ['/docs/dev/releases/release-rc.3/','/docs/dev/releases/rc3/'] +summary: TiDB RC3, released on June 16, 2017, focuses on MySQL compatibility, SQL optimization, stability, and performance. Highlights include refined privilege management, accelerated DDL, optimized load balancing, and open-sourced TiDB Ansible for easy cluster management. Detailed updates for TiDB, Placement Driver (PD), and TiKV include improved SQL query optimization, complete privilege management, support for HTTP API, system variables for query concurrency control, and more efficient data balance. PD supports gRPC, disaster recovery toolkit, and hot Region scheduling. TiKV supports gRPC, SST format snapshot, memory leak detection, and improved data importing speed. Overall, the release enhances performance, stability, and management capabilities. --- # TiDB RC3 Release Notes diff --git a/releases/release-rc.4.md b/releases/release-rc.4.md index 007b4b5e0ad41..f6bf730f51b39 100644 --- a/releases/release-rc.4.md +++ b/releases/release-rc.4.md @@ -1,6 +1,6 @@ --- title: TiDB RC4 Release Notes -aliases: ['/docs/dev/releases/release-rc.4/','/docs/dev/releases/rc4/'] +summary: TiDB RC4 is released with a focus on MySQL compatibility, SQL optimization, stability, and performance. Highlights include improved write performance, better query cost estimating, and support for TiSpark to access data in TiKV. Detailed updates include refactoring of the SQL query optimizer, support for JSON type and operations, and optimization of the scheduler in Placement Driver. TiKV now supports RC isolation level, Document Store, and more pushdown functions in Coprocessor. TiSpark beta release includes prediction pushdown, aggregation pushdown, and range pruning, capable of running a full set of TPC-H queries. --- # TiDB RC4 Release Notes diff --git a/releases/release-timeline.md b/releases/release-timeline.md index d16db271cca5a..606c5d8170b22 100644 --- a/releases/release-timeline.md +++ b/releases/release-timeline.md @@ -5,10 +5,31 @@ summary: Learn about the TiDB release timeline. # TiDB Release Timeline + + This document shows all the released TiDB versions in reverse chronological order. | Version | Release Date | | :--- | :--- | +| [7.5.7](/releases/release-7.5.7.md) | 2025-09-04 | +| [7.5.6](/releases/release-7.5.6.md) | 2025-03-14 | +| [6.5.12](/releases/release-6.5.12.md) | 2025-02-27 | +| [7.5.5](/releases/release-7.5.5.md) | 2024-12-31 | +| [7.1.6](/releases/release-7.1.6.md) | 2024-11-21 | +| [7.5.4](/releases/release-7.5.4.md) | 2024-10-15 | +| [6.5.11](/releases/release-6.5.11.md) | 2024-09-20 | +| [7.5.3](/releases/release-7.5.3.md) | 2024-08-05 | +| [6.5.10](/releases/release-6.5.10.md) | 2024-06-20 | +| [7.5.2](/releases/release-7.5.2.md) | 2024-06-13 | +| [7.1.5](/releases/release-7.1.5.md) | 2024-04-26 | +| [6.5.9](/releases/release-6.5.9.md) | 2024-04-12 | +| [7.1.4](/releases/release-7.1.4.md) | 2024-03-11 | +| [7.5.1](/releases/release-7.5.1.md) | 2024-02-29 | +| [6.5.8](/releases/release-6.5.8.md) | 2024-02-02 | +| [6.5.7](/releases/release-6.5.7.md) | 2024-01-08 | +| [7.1.3](/releases/release-7.1.3.md) | 2023-12-21 | +| [6.5.6](/releases/release-6.5.6.md) | 2023-12-07 | +| [7.5.0](/releases/release-7.5.0.md) | 2023-12-01 | | [7.1.2](/releases/release-7.1.2.md) | 2023-10-25 | | [7.4.0-DMR](/releases/release-7.4.0.md) | 2023-10-12 | | [6.5.5](/releases/release-6.5.5.md) | 2023-09-21 | @@ -176,4 +197,4 @@ This document shows all the released TiDB versions in reverse chronological orde | [rc4](/releases/release-rc.4.md) | 2017-08-04 | | [rc3](/releases/release-rc.3.md) | 2017-06-16 | | [rc2](/releases/release-rc.2.md) | 2017-03-01 | -| [rc1](/releases/release-rc.1.md) | 2016-12-23 | \ No newline at end of file +| [rc1](/releases/release-rc.1.md) | 2016-12-23 | diff --git a/releases/versioning.md b/releases/versioning.md index 63d933739a128..a014db7b39a77 100644 --- a/releases/versioning.md +++ b/releases/versioning.md @@ -16,7 +16,7 @@ TiDB offers two release series: * Long-Term Support Releases * Development Milestone Releases (introduced in TiDB v6.0.0) -To learn about the support policy for major releases of TiDB, see [TiDB Release Support Policy](https://en.pingcap.com/tidb-release-support-policy/). +To learn about the support policy for major releases of TiDB, see [TiDB Release Support Policy](https://www.pingcap.com/tidb-release-support-policy/). ## Release versioning diff --git a/replicate-between-primary-and-secondary-clusters.md b/replicate-between-primary-and-secondary-clusters.md index c4ccb346ef1b2..9b50a80bb688b 100644 --- a/replicate-between-primary-and-secondary-clusters.md +++ b/replicate-between-primary-and-secondary-clusters.md @@ -1,7 +1,6 @@ --- title: Replicate data between primary and secondary clusters summary: Learn how to replicate data from a primary cluster to a secondary cluster. -aliases: ['/docs/dev/incremental-replication-between-clusters/', '/tidb/dev/replicate-betwwen-primary-and-secondary-clusters/'] --- # Replicate Data Between Primary and Secondary Clusters @@ -99,12 +98,12 @@ To replicate incremental data from a running TiDB cluster to its secondary clust ## Step 2. Migrate full data -After setting up the environment, you can use the backup and restore functions of [BR](https://github.com/pingcap/tidb/tree/master/br)) to migrate full data. BR can be started in [three ways](/br/br-use-overview.md#deploy-and-use-br). In this document, we use the SQL statements, `BACKUP` and `RESTORE`. +After setting up the environment, you can use the backup and restore functions of [BR](https://github.com/pingcap/tidb/tree/release-7.5/br) to migrate full data. BR can be started in [three ways](/br/br-use-overview.md#deploy-and-use-br). In this document, we use the SQL statements, `BACKUP` and `RESTORE`. > **Note:** > +> - `BACKUP` and `RESTORE` SQL statements are experimental. It is not recommended that you use them in the production environment. They might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. > - In production clusters, performing a backup with GC disabled might affect cluster performance. It is recommended that you back up data in off-peak hours, and set RATE_LIMIT to a proper value to avoid performance degradation. -> > - If the versions of the upstream and downstream clusters are different, you should check [BR compatibility](/br/backup-and-restore-overview.md#some-tips). In this document, we assume that the upstream and downstream clusters are the same version. 1. Disable GC. diff --git a/replicate-data-to-kafka.md b/replicate-data-to-kafka.md index 71d4fad32f748..1cf94d1c41252 100644 --- a/replicate-data-to-kafka.md +++ b/replicate-data-to-kafka.md @@ -31,7 +31,7 @@ The preceding steps are performed in a lab environment. You can also deploy a cl 2. Create a Kafka cluster. - - Lab environment: refer to [Apache Kakfa Quickstart](https://kafka.apache.org/quickstart) to start a Kafka cluster. + - Lab environment: refer to [Apache Kafka Quickstart](https://kafka.apache.org/quickstart) to start a Kafka cluster. - Production environment: refer to [Running Kafka in Production](https://docs.confluent.io/platform/current/kafka/deployment.html) to deploy a Kafka production cluster. 3. (Optional) Create a Flink cluster. @@ -57,7 +57,7 @@ The preceding steps are performed in a lab environment. You can also deploy a cl 2. Create a changefeed to replicate incremental data to Kafka: ```shell - tiup ctl:v cdc changefeed create --server="http://127.0.0.1:8300" --sink-uri="kafka://127.0.0.1:9092/kafka-topic-name?protocol=canal-json" --changefeed-id="kafka-changefeed" --config="changefeed.conf" + tiup cdc:v cli changefeed create --server="http://127.0.0.1:8300" --sink-uri="kafka://127.0.0.1:9092/kafka-topic-name?protocol=canal-json" --changefeed-id="kafka-changefeed" --config="changefeed.conf" ``` - If the changefeed is successfully created, changefeed information, such as changefeed ID, is displayed, as shown below: @@ -73,13 +73,13 @@ The preceding steps are performed in a lab environment. You can also deploy a cl In a production environment, a Kafka cluster has multiple broker nodes. Therefore, you can add the addresses of multiple brokers to the sink UIR. This ensures stable access to the Kafka cluster. When the Kafka cluster is down, the changefeed still works. Suppose that a Kafka cluster has three broker nodes, with IP addresses being 127.0.0.1:9092, 127.0.0.2:9092, and 127.0.0.3:9092, respectively. You can create a changefeed with the following sink URI. ```shell - tiup ctl:v cdc changefeed create --server="http://127.0.0.1:8300" --sink-uri="kafka://127.0.0.1:9092,127.0.0.2:9092,127.0.0.3:9092/kafka-topic-name?protocol=canal-json&partition-num=3&replication-factor=1&max-message-bytes=1048576" --config="changefeed.conf" + tiup cdc:v cli changefeed create --server="http://127.0.0.1:8300" --sink-uri="kafka://127.0.0.1:9092,127.0.0.2:9092,127.0.0.3:9092/kafka-topic-name?protocol=canal-json&partition-num=3&replication-factor=1&max-message-bytes=1048576" --config="changefeed.conf" ``` 3. After creating the changefeed, run the following command to check the changefeed status: ```shell - tiup ctl:v cdc changefeed list --server="http://127.0.0.1:8300" + tiup cdc:v cli changefeed list --server="http://127.0.0.1:8300" ``` You can refer to [Manage TiCDC Changefeeds](/ticdc/ticdc-manage-changefeed.md) to manage the changefeed. diff --git a/role-based-access-control.md b/role-based-access-control.md index 4bb9811c26e58..efcb70c4f7ae5 100644 --- a/role-based-access-control.md +++ b/role-based-access-control.md @@ -1,7 +1,6 @@ --- title: Role-Based Access Control summary: This document introduces TiDB RBAC operations and implementation. -aliases: ['/docs/dev/role-based-access-control/','/docs/dev/reference/security/role-based-access-control/'] --- # Role-Based Access Control diff --git a/runtime-filter.md b/runtime-filter.md index c4d01012a35d2..1d2031322929f 100644 --- a/runtime-filter.md +++ b/runtime-filter.md @@ -67,7 +67,7 @@ The execution process of Runtime Filter is as follows: | | filter data | | | | +-----+----v------+ +-------+--------+ - | TableFullScan | | TabelFullScan | + | TableFullScan | | TableFullScan | | store_sales | | date_dim | +-----------------+ +----------------+ ``` diff --git a/scale-tidb-using-tiup.md b/scale-tidb-using-tiup.md index d86a49fc442db..d3bf5834fb7d0 100644 --- a/scale-tidb-using-tiup.md +++ b/scale-tidb-using-tiup.md @@ -1,7 +1,6 @@ --- title: Scale a TiDB Cluster Using TiUP summary: Learn how to scale the TiDB cluster using TiUP. -aliases: ['/docs/dev/scale-tidb-using-tiup/','/docs/dev/how-to/scale/with-tiup/'] --- # Scale a TiDB Cluster Using TiUP @@ -127,7 +126,26 @@ This section exemplifies how to add a TiDB node to the `10.0.1.5` host. If you see `Scaled cluster out successfully`, the scale-out operation succeeds. -3. Check the cluster status: +3. Refresh the cluster configuration. + + > **Note:** + > + > - Refreshing cluster configuration is only required after you add PD nodes. If you only add TiDB or TiKV nodes, skip this step. + > - If you are using TiUP v1.15.0 or a later version, skip this step because TiUP does it. If you are using a TiUP version earlier than v1.15.0, perform the following sub-steps. + + 1. Refresh the cluster configuration: + + ```shell + tiup cluster reload --skip-restart + ``` + + 2. Refresh the Prometheus configuration and restart Prometheus: + + ```shell + tiup cluster reload -R prometheus + ``` + +4. Check the cluster status: {{< copyable "shell-regular" >}} @@ -276,7 +294,7 @@ This section exemplifies how to remove a TiKV node from the `10.0.1.5` host. ``` Starting /root/.tiup/components/cluster/v1.12.3/cluster display TiDB Cluster: - TiDB Version: v7.4.0 + TiDB Version: v{{{ .tidb-version }}} ID Role Host Ports Status Data Dir Deploy Dir -- ---- ---- ----- ------ -------- ---------- 10.0.1.3:8300 cdc 10.0.1.3 8300 Up data/cdc-8300 deploy/cdc-8300 @@ -307,7 +325,26 @@ This section exemplifies how to remove a TiKV node from the `10.0.1.5` host. If you see `Scaled cluster in successfully`, the scale-in operation succeeds. -3. Check the cluster status: +3. Refresh the cluster configuration. + + > **Note:** + > + > - Refreshing cluster configuration is only required after you remove PD nodes. If you only remove TiDB or TiKV nodes, skip this step. + > - If you are using TiUP v1.15.0 or a later version, skip this step because TiUP does it. If you are using a TiUP version earlier than v1.15.0, perform the following sub-steps. + + 1. Refresh the cluster configuration: + + ```shell + tiup cluster reload --skip-restart + ``` + + 2. Refresh the Prometheus configuration and restart Prometheus: + + ```shell + tiup cluster reload -R prometheus + ``` + +4. Check the cluster status: The scale-in process takes some time. You can run the following command to check the scale-in status: diff --git a/schedule-replicas-by-topology-labels.md b/schedule-replicas-by-topology-labels.md index 9604dd74f371d..ad5f7ee135f3c 100644 --- a/schedule-replicas-by-topology-labels.md +++ b/schedule-replicas-by-topology-labels.md @@ -1,7 +1,6 @@ --- title: Schedule Replicas by Topology Labels summary: Learn how to schedule replicas by topology labels. -aliases: ['/docs/dev/location-awareness/','/docs/dev/how-to/deploy/geographic-redundancy/location-awareness/','/tidb/dev/location-awareness'] --- # Schedule Replicas by Topology Labels @@ -14,9 +13,156 @@ To improve the high availability and disaster recovery capability of TiDB cluste To make this mechanism effective, you need to properly configure TiKV and PD so that the topology information of the cluster, especially the TiKV location information, is reported to PD during deployment. Before you begin, see [Deploy TiDB Using TiUP](/production-deployment-using-tiup.md) first. -## Configure `labels` based on the cluster topology +## Configure `labels` for TiKV, TiFlash, and TiDB -### Configure `labels` for TiKV and TiFlash +You can configure `labels` for TiKV, TiFlash, and TiDB based on the cluster topology. + +### Configure a cluster using TiUP (recommended) + +When using TiUP to deploy a cluster, you can configure the TiKV location in the [initialization configuration file](/production-deployment-using-tiup.md#step-3-initialize-cluster-topology-file). TiUP will generate the corresponding configuration files for TiDB, TiKV, PD, and TiFlash during deployment. + +In the following example, a two-layer topology of `zone/host` is defined. The TiDB nodes, TiKV nodes, and TiFlash nodes of the cluster are distributed among three zones, z1, z2, and z3. + +- In each zone, there are two hosts that have TiDB instances deployed, and each host has a separate TiDB instance deployed. +- In each zone, there are two hosts that have TiKV instances deployed. In z1, each host has two TiKV instances deployed. In z2 and z3, each host has a separate TiKV instance deployed. +- In each zone, there are two hosts that have TiFlash instances deployed, and each host has a separate TiFlash instance deployed. + +In the following example, `tidb-host-machine-n` represents the IP address of the `n`th TiDB node, `tikv-host-machine-n` represents the IP address of the `n`th TiKV node, and `tiflash-host-machine-n` represents the IP address of the `n`th TiFlash node. + +``` +server_configs: + pd: + replication.location-labels: ["zone", "host"] +tidb_servers: +# z1 + - host: tidb-host-machine-1 + config: + labels: + zone: z1 + host: tidb-host-machine-1 + - host: tidb-host-machine-2 + config: + labels: + zone: z1 + host: tidb-host-machine-2 +# z2 + - host: tidb-host-machine-3 + config: + labels: + zone: z2 + host: tidb-host-machine-3 + - host: tikv-host-machine-4 + config: + labels: + zone: z2 + host: tidb-host-machine-4 +# z3 + - host: tidb-host-machine-5 + config: + labels: + zone: z3 + host: tidb-host-machine-5 + - host: tidb-host-machine-6 + config: + labels: + zone: z3 + host: tidb-host-machine-6 +tikv_servers: +# z1 + # machine-1 on z1 + - host: tikv-host-machine-1 + port:20160 + config: + server.labels: + zone: z1 + host: tikv-host-machine-1 + - host: tikv-host-machine-1 + port:20161 + config: + server.labels: + zone: z1 + host: tikv-host-machine-1 + # machine-2 on z1 + - host: tikv-host-machine-2 + port:20160 + config: + server.labels: + zone: z1 + host: tikv-host-machine-2 + - host: tikv-host-machine-2 + port:20161 + config: + server.labels: + zone: z1 + host: tikv-host-machine-2 +# z2 + - host: tikv-host-machine-3 + config: + server.labels: + zone: z2 + host: tikv-host-machine-3 + - host: tikv-host-machine-4 + config: + server.labels: + zone: z2 + host: tikv-host-machine-4 +# z3 + - host: tikv-host-machine-5 + config: + server.labels: + zone: z3 + host: tikv-host-machine-5 + - host: tikv-host-machine-6 + config: + server.labels: + zone: z3 + host: tikv-host-machine-6 + +tiflash_servers: +# z1 + - host: tiflash-host-machine-1 + learner_config: + server.labels: + zone: z1 + host: tiflash-host-machine-1 + - host: tiflash-host-machine-2 + learner_config: + server.labels: + zone: z1 + host: tiflash-host-machine-2 +# z2 + - host: tiflash-host-machine-3 + learner_config: + server.labels: + zone: z2 + host: tiflash-host-machine-3 + - host: tiflash-host-machine-4 + learner_config: + server.labels: + zone: z2 + host: tiflash-host-machine-4 +# z3 + - host: tiflash-host-machine-5 + learner_config: + server.labels: + zone: z3 + host: tiflash-host-machine-5 + - host: tiflash-host-machine-6 + learner_config: + server.labels: + zone: z3 + host: tiflash-host-machine-6 +``` + +For details, see [Geo-distributed Deployment topology](/geo-distributed-deployment-topology.md). + +> **Note:** +> +> If you have not configured `replication.location-labels` in the configuration file, when you deploy a cluster using this topology file, an error might occur. It is recommended that you confirm `replication.location-labels` is configured in the configuration file before deploying a cluster. + +### Configure a cluster using command lines or configuration files + +#### Configure `labels` for TiKV and TiFlash You can use the command-line flag or set the TiKV or TiFlash configuration file to bind some attributes in the form of key-value pairs. These attributes are called `labels`. After TiKV and TiFlash are started, they report their `labels` to PD so users can identify the location of TiKV and TiFlash nodes. @@ -50,7 +196,7 @@ rack = "" host = "" ``` -### (Optional) Configure `labels` for TiDB +#### (Optional) Configure `labels` for TiDB When [Follower read](/follower-read.md) is enabled, if you want TiDB to prefer to read data from the same region, you need to configure `labels` for TiDB nodes. @@ -68,7 +214,7 @@ host = "" > > Currently, TiDB depends on the `zone` label to match and select replicas that are in the same region. To use this feature, you need to include `zone` when [configuring `location-labels` for PD](#configure-location-labels-for-pd), and configure `zone` when configuring `labels` for TiDB, TiKV, and TiFlash. For more details, see [Configure `labels` for TiKV and TiFlash](#configure-labels-for-tikv-and-tiflash). -### Configure `location-labels` for PD +## Configure `location-labels` for PD According to the description above, the label can be any key-value pair used to describe TiKV attributes. But PD cannot identify the location-related labels and the layer relationship of these labels. Therefore, you need to make the following configuration for PD to understand the TiKV node topology. @@ -100,7 +246,7 @@ To configure `location-labels`, choose one of the following methods according to pd-ctl config set location-labels zone,rack,host ``` -### Configure `isolation-level` for PD +## Configure `isolation-level` for PD If `location-labels` has been configured, you can further enhance the topological isolation requirements on TiKV clusters by configuring `isolation-level` in the PD configuration file. @@ -127,115 +273,6 @@ The `location-level` configuration is an array of strings, which needs to corres > > `isolation-level` is empty by default, which means there is no mandatory restriction on the isolation level. To set it, you need to configure `location-labels` for PD and ensure that the value of `isolation-level` is one of `location-labels` names. -### Configure a cluster using TiUP (recommended) - -When using TiUP to deploy a cluster, you can configure the TiKV location in the [initialization configuration file](/production-deployment-using-tiup.md#step-3-initialize-cluster-topology-file). TiUP will generate the corresponding configuration files for TiKV, PD, and TiFlash during deployment. - -In the following example, a two-layer topology of `zone/host` is defined. The TiKV nodes and TiFlash nodes of the cluster are distributed among three zones, z1, z2, and z3. - -- In each zone, there are two hosts that have TiKV instances deployed. In z1, each host has two TiKV instances deployed. In z2 and z3, each host has a separate TiKV instance deployed. -- In each zone, there are two hosts that have TiFlash instances deployed, and each host has a separate TiFlash instance deployed. - -In the following example, `tikv-host-machine-n` represents the IP address of the `n`th TiKV node, and `tiflash-host-machine-n` represents the IP address of the `n`th TiFlash node. - -``` -server_configs: - pd: - replication.location-labels: ["zone", "host"] - -tikv_servers: -# z1 - # machine-1 on z1 - - host: tikv-host-machine-1 - port:20160 - config: - server.labels: - zone: z1 - host: tikv-host-machine-1 - - host: tikv-host-machine-1 - port:20161 - config: - server.labels: - zone: z1 - host: tikv-host-machine-1 - # machine-2 on z1 - - host: tikv-host-machine-2 - port:20160 - config: - server.labels: - zone: z1 - host: tikv-host-machine-2 - - host: tikv-host-machine-2 - port:20161 - config: - server.labels: - zone: z1 - host: tikv-host-machine-2 -# z2 - - host: tikv-host-machine-3 - config: - server.labels: - zone: z2 - host: tikv-host-machine-3 - - host: tikv-host-machine-4 - config: - server.labels: - zone: z2 - host: tikv-host-machine-4 -# z3 - - host: tikv-host-machine-5 - config: - server.labels: - zone: z3 - host: tikv-host-machine-5 - - host: tikv-host-machine-6 - config: - server.labels: - zone: z3 - host: tikv-host-machine-6 - -tiflash_servers: -# z1 - - host: tiflash-host-machine-1 - learner_config: - server.labels: - zone: z1 - host: tiflash-host-machine-1 - - host: tiflash-host-machine-2 - learner_config: - server.labels: - zone: z1 - host: tiflash-host-machine-2 -# z2 - - host: tiflash-host-machine-3 - learner_config: - server.labels: - zone: z2 - host: tiflash-host-machine-3 - - host: tiflash-host-machine-4 - learner_config: - server.labels: - zone: z2 - host: tiflash-host-machine-4 -# z3 - - host: tiflash-host-machine-5 - learner_config: - server.labels: - zone: z3 - host: tiflash-host-machine-5 - - host: tiflash-host-machine-6 - learner_config: - server.labels: - zone: z3 - host: tiflash-host-machine-6 -``` - -For details, see [Geo-distributed Deployment topology](/geo-distributed-deployment-topology.md). - -> **Note:** -> -> If you have not configured `replication.location-labels` in the configuration file, when you deploy a cluster using this topology file, an error might occur. It is recommended that you confirm `replication.location-labels` is configured in the configuration file before deploying a cluster. - ## PD schedules based on topology label PD schedules replicas according to the label layer to make sure that different replicas of the same data are scattered as much as possible. diff --git a/schema-object-names.md b/schema-object-names.md index bb216ad01c870..01ce379e9f482 100644 --- a/schema-object-names.md +++ b/schema-object-names.md @@ -1,7 +1,6 @@ --- title: Schema Object Names summary: Learn about schema object names in TiDB SQL statements. -aliases: ['/docs/dev/schema-object-names/','/docs/dev/reference/sql/language-structure/schema-object-names/'] --- # Schema Object Names diff --git a/scripts/check-keywords.sh b/scripts/check-keywords.sh index 6df1183902982..fcee793d7fa87 100755 --- a/scripts/check-keywords.sh +++ b/scripts/check-keywords.sh @@ -16,33 +16,45 @@ keywords = kwdocs.read_text() errors = 0 section = "Unknown" for line in parser.read_text().split("\n"): - if line.find("The following tokens belong to ReservedKeyword") >= 0: + if line == "": + section = "NotKeywordToken" + + elif line.find("The following tokens belong to ReservedKeyword") >= 0: section = "ReservedKeyword" - if line.find("The following tokens belong to UnReservedKeyword") >= 0: + elif line.find("The following tokens belong to UnReservedKeyword") >= 0: section = "UnReservedKeyword" - if line.find("The following tokens belong to NotKeywordToken") >= 0: + elif line.find("The following tokens belong to TiDBKeyword") >= 0: + section = "TiDBKeyword" + + elif line.find("The following tokens belong to NotKeywordToken") >= 0: section = "NotKeywordToken" if section == "ReservedKeyword": if m := re.match(r'^\t\w+\s+"(\w+)"$', line): kw = m.groups()[0] if not ( - kwm := re.search(f"^- {kw} \((R|R-Window)\)$", keywords, re.MULTILINE) + kwm := re.search(f"^- {kw} \\((R|R-Window)\\)$", keywords, re.MULTILINE) ): if kwm := re.search(f"^- {kw}$", keywords, re.MULTILINE): - print(f"Keyword not labeled as reserved: {kw}") - errors += 1 + print(f"Reserved keyword not labeled as reserved: {kw}") else: print(f"Missing docs for reserved keyword: {kw}") - errors += 1 + errors += 1 - if section == "UnReservedKeyword": + if section in ["UnReservedKeyword", "TiDBKeyword"]: if m := re.match(r'^\t\w+\s+"(\w+)"$', line): kw = m.groups()[0] if not (kwm := re.search(f"^- {kw}$", keywords, re.MULTILINE)): - print(f"Missing docs for non-reserved keyword: {kw}") + if kwm := re.search( + f"^- {kw} \\((R|R-Window)\\)$", keywords, re.MULTILINE + ): + print( + f"Non-reserved keyword from {section} labeled as reserved: {kw}" + ) + else: + print(f"Missing docs for non-reserved keyword from {section}: {kw}") errors += 1 sys.exit(errors) diff --git a/scripts/concatMdByToc.js b/scripts/concatMdByToc.js index 1875a3fb5cbb6..0d8d9dbbe4fcc 100644 --- a/scripts/concatMdByToc.js +++ b/scripts/concatMdByToc.js @@ -35,8 +35,26 @@ const isFileExist = (path = "") => { return fs.existsSync(path); }; +const variablePattern = /{{{\s*\.(.+?)\s*}}}/g; + +function getValueByPath(obj, path) { + return path.split(".").reduce((acc, key) => (acc ? acc[key] : ""), obj) ?? ""; +} + +const replaceVariables = (content, variables) => { + return content.replace(variablePattern, (match, path) => { + const value = getValueByPath(variables, path.trim()); + if (value === "") { + return match; + } + return String(value); + }); +}; + +const headingHashReg = /\s*\{#([a-zA-Z0-9\-_]+)\}$/; + const handleMdAst = (mdAst, fileName = "") => { - visit(mdAst, (node) => { + visit(mdAst, (node, index, parent) => { switch (node.type) { case "yaml": const fileNameWithoutExt = fileName @@ -65,6 +83,28 @@ const handleMdAst = (mdAst, fileName = "") => { node.url = `#title-${mdName}`; } break; + case "heading": + const lastChild = node.children?.[node.children.length - 1]; + if ( + lastChild && + lastChild.type === "text" && + headingHashReg.test(lastChild.value) + ) { + const match = lastChild.value.match(headingHashReg); + const hash = match[1]; + + lastChild.value = lastChild.value.replace(headingHashReg, ""); + + const anchorNode = { + type: "html", + value: ``, + }; + + if (parent && Array.isArray(parent.children)) { + parent.children.splice(index, 0, anchorNode); + } + } + break; default: // console.log(node); break; @@ -78,19 +118,23 @@ const handleSingleMd = (filePath) => { const mdAst = generateMdAstFromFile(mdFileContent); handleMdAst(mdAst, fileName); const MdStr = astNode2mdStr(mdAst); - const newMdStr = MdStr.replaceAll(copyableReg, ""); - isFileExist(`tmp/${targetFile}`) - ? fs.appendFileSync(`tmp/${targetFile}`, newMdStr) - : writeFileSync(`tmp/${targetFile}`, newMdStr); + return MdStr.replaceAll(copyableReg, ""); }; const main = () => { const fileList = getAllMdList(srcToc); - // console.log(fileList); - // handleSingleMd("./overview.md"); + let mergedStr = ""; + fileList.forEach((filePath) => { - handleSingleMd(`.${filePath}`); + mergedStr += `${handleSingleMd(`.${filePath}`)}\n\n`; }); + + const variables = JSON.parse( + fs.readFileSync(path.resolve(__dirname, "../variables.json"), "utf8") + ); + mergedStr = replaceVariables(mergedStr, variables); + + writeFileSync(`tmp/${targetFile}`, mergedStr); }; main(); diff --git a/scripts/generate_pdf.sh b/scripts/generate_pdf.sh index fb2cf3e5225d3..a780399285bcf 100755 --- a/scripts/generate_pdf.sh +++ b/scripts/generate_pdf.sh @@ -17,6 +17,7 @@ _version_tag="$(date '+%Y%m%d')" pandoc -N --toc --smart --latex-engine=xelatex \ --template=templates/template.tex \ +--include-in-header=templates/deeplist.tex \ --columns=80 \ --listings \ -V title="TiDB Documentation" \ @@ -26,4 +27,4 @@ pandoc -N --toc --smart --latex-engine=xelatex \ -V fontsize=12pt \ -V geometry:margin=1in \ -V include-after="\\input{templates/copyright.tex}" \ -"doc.md" -s -o "output.pdf" \ No newline at end of file +"doc.md" -s -o "output.pdf" diff --git a/scripts/merge_by_toc.py b/scripts/merge_by_toc.py index 8a1124c61b45b..d00d1913de68c 100755 --- a/scripts/merge_by_toc.py +++ b/scripts/merge_by_toc.py @@ -10,6 +10,8 @@ import re import os import sys +import json +import unicodedata followups = [] in_toc = False @@ -91,6 +93,69 @@ tag = tag[3:] file_link_name[f] = tag.lower().replace(" ", "-") +def load_variables(): + current_dir = os.path.dirname(os.path.abspath(__file__)) + variables_path = os.path.join(current_dir, "../variables.json") + try: + with open(variables_path, "r", encoding="utf-8") as f: + return json.load(f) + except Exception as e: + return {} +variable_pattern = re.compile(r"{{{\s*\.(.+?)\s*}}}") + +def get_value_by_path(obj, path): + keys = path.split(".") + for key in keys: + if isinstance(obj, dict) and key in obj: + obj = obj[key] + else: + return "" + return str(obj) + +def replace_variables(text, variables): + def replacer(match): + path = match.group(1).strip() + value = get_value_by_path(variables, path) + return str(value) if value != "" else match.group(0) + return variable_pattern.sub(replacer, text) + +def slugify(title): + slug = title.strip().lower() + slug = unicodedata.normalize('NFKD', slug) + slug = re.sub(r"[^\w\s-]", "", slug) # remove punctuation + slug = re.sub(r"[\s_]+", "-", slug) # spaces and underscores to dash + return slug + +custom_id_map = {} # key = custom-id, value = slugified title + +heading_with_custom_id_pattern = re.compile(r"^(#+)\s+(.*?)(?:\s+\{#([^\}]+)\})?$", re.MULTILINE) + +def extract_custom_ids_and_clean(chapter): + def repl(match): + hashes = match.group(1) + title = match.group(2).strip() + custom_id = match.group(3) + + if custom_id: + anchor = slugify(title) + custom_id_map[custom_id] = anchor + return f"{hashes} {title}" # remove the `{#...}` + else: + return match.group(0) + + return heading_with_custom_id_pattern.sub(repl, chapter) + +def replace_custom_id_links(content): + # [text](/path#custom-id) → [text](#anchor-text) + def repl(match): + text, url, frag = match.group(1), match.group(2), match.group(3) + if frag and frag.startswith("#"): + cid = frag[1:] + if cid in custom_id_map: + return f"[{text}](#{custom_id_map[cid]})" + return match.group(0) + + return hyper_link_pattern.sub(repl, content) def replace_link_wrap(chapter, name): @@ -140,6 +205,7 @@ def replace_heading(match): def remove_copyable(match): return "" +variables = load_variables() # stage 3, concat files for type_, level, name in followups: @@ -151,9 +217,11 @@ def remove_copyable(match): try: with open(name) as fp: chapter = fp.read() + chapter = replace_variables(chapter, variables) chapter = replace_link_wrap(chapter, name) chapter = copyable_snippet_pattern.sub(remove_copyable, chapter) - + chapter = extract_custom_ids_and_clean(chapter) + chapter = replace_custom_id_links(chapter) # This block is to filter xxx try: custom_content_platform = sys.argv[3] diff --git a/scripts/upload.py b/scripts/upload.py deleted file mode 100755 index 7ce7115ae1296..0000000000000 --- a/scripts/upload.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 -#-*- coding:utf-8 -*- - -import sys -import os -from qiniu import Auth, put_file, etag, urlsafe_base64_encode -import qiniu.config -from qiniu.compat import is_py2, is_py3 - -import logging -import boto3 -from botocore.exceptions import ClientError - -QINIU_ACCESS_KEY = os.getenv('QINIU_ACCESS_KEY') -QINIU_SECRET_KEY = os.getenv('QINIU_SECRET_KEY') -QINIU_BUCKET_NAME = os.getenv('QINIU_BUCKET_NAME') -AWS_BUCKET_NAME = os.getenv('AWS_BUCKET_NAME') - -assert(QINIU_ACCESS_KEY and QINIU_SECRET_KEY and QINIU_BUCKET_NAME) - -def progress_handler(progress, total): - print("{}/{} {:.2f}".format(progress, total, progress/total*100)) - -# local_file: local file path -# remote_name: 上传到七牛后保存的文件名 -def upload_to_qiniu(local_file, remote_name, ttl=3600): - print('uploading to qiniu', local_file, remote_name, ttl) - #构建鉴权对象 - q = Auth(QINIU_ACCESS_KEY, QINIU_SECRET_KEY) - - #生成上传 Token,可以指定过期时间等 - token = q.upload_token(QINIU_BUCKET_NAME, remote_name, ttl) - - ret, info = put_file(token, remote_name, local_file) - print("ret", ret) - print("info", info) - # assert ret['key'] == remote_name - if is_py2: - assert ret['key'].encode('utf-8') == remote_name - elif is_py3: - assert ret['key'] == remote_name - - assert ret['hash'] == etag(local_file) - -def upload_to_aws(local_file, remote_name=None): - print('uploading to aws') - - """Upload a file to an S3 bucket - :param local_file: File to upload - :param remote_name: S3 object name. If not specified then local_file is used - :return: True if file was uploaded, else False - """ - - # If S3 remote_name was not specified, use local_file - if remote_name is None: - remote_name = local_file - - # Upload the file - s3_client = boto3.client('s3') - try: - response = s3_client.upload_file(local_file, AWS_BUCKET_NAME, remote_name) - except ClientError as e: - logging.error(e) - return False - return True - -if __name__ == "__main__": - local_file = sys.argv[1] - remote_name = sys.argv[2] - print(local_file, remote_name) - #upload_to_aws(local_file, remote_name) - upload_to_qiniu(local_file, remote_name) - - print("https://download.pingcap.org/{}".format(remote_name)) diff --git a/scripts/verify-links.sh b/scripts/verify-links.sh index 70b5b38c8aba6..119eb1fec53be 100755 --- a/scripts/verify-links.sh +++ b/scripts/verify-links.sh @@ -2,7 +2,7 @@ # # This script is used to verify links in markdown docs. # -# - External links are ignored in CI because these links may go down out of our contorl. +# - External links are ignored in CI because these links may go down out of our control. # - Anchors are ignored # - Internal links conventions # - Must be absolute and start from repo root diff --git a/security-compatibility-with-mysql.md b/security-compatibility-with-mysql.md index e96c74c830afa..7c034edf95765 100644 --- a/security-compatibility-with-mysql.md +++ b/security-compatibility-with-mysql.md @@ -1,7 +1,6 @@ --- title: Security Compatibility with MySQL summary: Learn TiDB's security compatibilities with MySQL. -aliases: ['/docs/dev/security-compatibility-with-mysql/','/docs/dev/reference/security/compatibility/'] --- # Security Compatibility with MySQL @@ -108,9 +107,9 @@ The implementation mechanisms are consistent between TiDB and MySQL. Both use th ## Authentication plugin status -TiDB supports multiple authentication methods. These methods can be specified on a per user basis using [`CREATE USER`](/sql-statements/sql-statement-create-user.md) and [`ALTER USER`](/sql-statements/sql-statement-create-user.md). These methods are compatible with the authentication methods of MySQL with the same names. +TiDB supports multiple authentication methods. These methods can be specified on a per user basis using [`CREATE USER`](/sql-statements/sql-statement-create-user.md) and [`ALTER USER`](/sql-statements/sql-statement-alter-user.md). These methods are compatible with the authentication methods of MySQL with the same names. -You can use one of the following supported authentication methods in the table. To specify a default method that the server advertises when the client-server connection is being established, set the [`default_authentication_plugin`](/system-variables.md#default_authentication_plugin) variable. `tidb_sm3_password` is the SM3 authentication method only supported in TiDB. Therefore, to authenticate using this method, you must connect to TiDB using [TiDB-JDBC](https://github.com/pingcap/mysql-connector-j/tree/release/8.0-sm3). `tidb_auth_token` is a JSON Web Token (JWT) based authentication method used only in TiDB Cloud. +You can use one of the following supported authentication methods in the table. To specify a default method that the server advertises when the client-server connection is being established, set the [`default_authentication_plugin`](/system-variables.md#default_authentication_plugin) variable. `tidb_sm3_password` is the SM3 authentication method only supported in TiDB. Therefore, to authenticate using this method, you must connect to TiDB using [TiDB-JDBC](https://github.com/pingcap/mysql-connector-j/tree/release/8.0-sm3). `tidb_auth_token` is a JSON Web Token (JWT)-based authentication method used in TiDB Cloud, and you can also configure it for use in TiDB Self-Managed. @@ -140,3 +139,122 @@ The support for TLS authentication is configured differently. For detailed infor | ed25519 (MariaDB) | No | | GSSAPI (MariaDB) | No | | FIDO | No | + +### `tidb_auth_token` + +`tidb_auth_token` is a passwordless authentication method based on [JSON Web Token (JWT)](https://datatracker.ietf.org/doc/html/rfc7519). In v6.4.0, `tidb_auth_token` is only used for user authentication in TiDB Cloud. Starting from v6.5.0, you can also configure `tidb_auth_token` as a user authentication method for TiDB Self-Managed. Different from password-based authentication methods such as `mysql_native_password` and `caching_sha2_password`, when you create users using `tidb_auth_token`, there is no need to set or store custom passwords. To log into TiDB, users only need to use a signed token instead of a password, which simplifies the authentication process and improves security. + +#### JWT + +JWT consists of three parts: Header, Payload, and Signature. After being encoded using base64, they are concatenated into a string separated by dots (`.`) for transmission between the client and server. + +The Header describes the metadata of the JWT, including 3 parameters: + +* `alg`: the algorithm for signature, which is `RS256` by default. +* `typ`: the type of token, which is `JWT`. +* `kid`: the key ID for generating token signature. + +Here is an example for Header: + +```json +{ + "alg": "RS256", + "kid": "the-key-id-0", + "typ": "JWT" +} +``` + +Payload is the main part of JWT, which stores the user information. Each field in the Payload is called a claim. The claims required for TiDB user authentication are as follows: + +* `iss`: if `TOKEN_ISSUER` is not specified or set to empty when [`CREATE USER`](/sql-statements/sql-statement-create-user.md), this claim is not required; otherwise, `iss` should use the same value as `TOKEN_ISSUER`. +* `sub`: this claim is required to be the same as the username to be authenticated. +* `iat`: it means `issued at`, the timestamp when the token is issued. In TiDB, this value must not be later than the authentication time or earlier than 15 minutes before authentication. +* `exp`: the timestamp when the token expires. If it is earlier than the time of authentication, the authentication fails. +* `email`: the email can be specified when creating a user by `ATTRIBUTE '{"email": "xxxx@pingcap.com"}`. If no email is specified when a user is created, this claim must be set as an empty string; otherwise, this claim must be the same as the specified value when the user is created. + +Here is an example for Payload: + +```json +{ + "email": "user@pingcap.com", + "exp": 1703305494, + "iat": 1703304594, + "iss": "issuer-abc", + "sub": "user@pingcap.com" +} +``` + +Signature is used to sign the Header and Payload data. + +> **Warning:** +> +> - The encoding of the Header and Payload in base64 is reversible. Do **Not** attach any sensitive information to them. +> - The `tidb_auth_token` authentication method requires clients to support the [`mysql_clear_password`](https://dev.mysql.com/doc/refman/8.0/en/cleartext-pluggable-authentication.html) plugin to send the token to TiDB in plain text. Therefore, you need to [enale TLS between clients and servers](/enable-tls-between-clients-and-servers.md) before using `tidb_auth_token`. + +#### Usage + +To configure and use `tidb_auth_token` as the authentication method for TiDB Self-Managed users, take the following steps: + +1. Configure [`auth-token-jwks`](/tidb-configuration-file.md#auth-token-jwks-new-in-v640) and [`auth-token-refresh-interval`](/tidb-configuration-file.md#auth-token-refresh-interval-new-in-v640) in the TiDB configuration file. + + For example, you can get an example JWKS using the following command: + + ```bash + wget https://raw.githubusercontent.com/CbcWestwolf/generate_jwt/master/JWKS.json + ``` + + Then, configure the path of the example JWKS in `config.toml`: + + ```toml + [security] + auth-token-jwks = "JWKS.json" + ``` + +2. Start `tidb-server` and periodically update and save the JWKS to the path specified by `auth-token-jwks`. + +3. Create a user with `tidb_auth_token`, and specify `iss` and `email` as needed using `REQUIRE TOKEN_ISSUER` and `ATTRIBUTE '{"email": "xxxx@pingcap.com"}`. + + For example, create a user `user@pingcap.com` with `tidb_auth_token`: + + ```sql + CREATE USER 'user@pingcap.com' IDENTIFIED WITH 'tidb_auth_token' REQUIRE TOKEN_ISSUER 'issuer-abc' ATTRIBUTE '{"email": "user@pingcap.com"}'; + ``` + +4. Generate and sign a token for authentication, and authenticate using the `mysql_clear_text` plugin of the MySQL client. + + Install the JWT generation tool via `go install github.com/cbcwestwolf/generate_jwt` (this tool is only used for testing `tidb_auth_token`). For example: + + ```text + generate_jwt --kid "the-key-id-0" --sub "user@pingcap.com" --email "user@pingcap.com" --iss "issuer-abc" + ``` + + It prints the public key and token as follows: + + ```text + -----BEGIN PUBLIC KEY----- + MIIBCgKCAQEAq8G5n9XBidxmBMVJKLOBsmdOHrCqGf17y9+VUXingwDUZxRp2Xbu + LZLbJtLgcln1lC0L9BsogrWf7+pDhAzWovO6Ai4Aybu00tJ2u0g4j1aLiDdsy0gy + vSb5FBoL08jFIH7t/JzMt4JpF487AjzvITwZZcnsrB9a9sdn2E5B/aZmpDGi2+Is + f5osnlw0zvveTwiMo9ba416VIzjntAVEvqMFHK7vyHqXbfqUPAyhjLO+iee99Tg5 + AlGfjo1s6FjeML4xX7sAMGEy8FVBWNfpRU7ryTWoSn2adzyA/FVmtBvJNQBCMrrA + hXDTMJ5FNi8zHhvzyBKHU0kBTS1UNUbP9wIDAQAB + -----END PUBLIC KEY----- + + eyJhbGciOiJSUzI1NiIsImtpZCI6InRoZS1rZXktaWQtMCIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InVzZXJAcGluZ2NhcC5jb20iLCJleHAiOjE3MDMzMDU0OTQsImlhdCI6MTcwMzMwNDU5NCwiaXNzIjoiaXNzdWVyLWFiYyIsInN1YiI6InVzZXJAcGluZ2NhcC5jb20ifQ.T4QPh2hTB5on5xCuvtWiZiDTuuKvckggNHtNaovm1F4RvwUv15GyOqj9yMstE-wSoV5eLEcPC2HgE6eN1C6yH_f4CU-A6n3dm9F1w-oLbjts7aYCl8OHycVYnq609fNnb8JLsQAmd1Zn9C0JW899-WSOQtvjLqVSPe9prH-cWaBVDQXzUJKxwywQzk9v-Z1Njt9H3Rn9vvwwJEEPI16VnaNK38I7YG-1LN4fAG9jZ6Zwvz7vb_s4TW7xccFf3dIhWTEwOQ5jDPCeYkwraRXU8NC6DPF_duSrYJc7d7Nu9Z2cr-E4i1Rt_IiRTuIIzzKlcQGg7jd9AGEfGe_SowsA-w + ``` + + Copy the preceding token in the last line for login: + + ```Shell + mycli -h 127.0.0.1 -P 4000 -u 'user@pingcap.com' -p '' + ``` + + Ensure that the MySQL client here supports the `mysql_clear_password` plugin. [mycli](https://www.mycli.net/) supports and enables this plugin by default. If you are using the [mysql command-line client](https://dev.mysql.com/doc/refman/8.0/en/mysql.html), you need to use the `--enable-cleartext-plugin` option to enable this plugin: + + ```Shell + mysql -h 127.0.0.1 -P 4000 -u 'user@pingcap.com' -p'' --enable-cleartext-plugin + ``` + + If an incorrect `--sub` is specified when the token is generated (such as `--sub "wronguser@pingcap.com"`), the authentication using this token would fail. + +You can encode and decode a token using the debugger provided by [jwt.io](https://jwt.io/). \ No newline at end of file diff --git a/shard-row-id-bits.md b/shard-row-id-bits.md index 3109bcc48e466..3c4d4f77fc14a 100644 --- a/shard-row-id-bits.md +++ b/shard-row-id-bits.md @@ -5,11 +5,11 @@ summary: Learn the SHARD_ROW_ID_BITS attribute. # SHARD_ROW_ID_BITS -This document introduces the `SHARD_ROW_ID_BITS` table attribute, which is used to set the number of bits of the shards after the implicit `_tidb_rowid` is sharded. +This document introduces the `SHARD_ROW_ID_BITS` table attribute, which is used to set the number of bits of the shards after the implicit [`_tidb_rowid`](https://docs.pingcap.com/tidb/v7.5/tidb-rowid/) is sharded. ## Concept -For the tables with a non-clustered primary key or no primary key, TiDB uses an implicit auto-increment row ID. When a large number of `INSERT` operations are performed, the data is written into a single Region, causing a write hot spot. +For tables with a non-clustered primary key or no primary key, TiDB uses the automatically generated [`_tidb_rowid`](https://docs.pingcap.com/tidb/v7.5/tidb-rowid/) as an implicit auto-increment row ID. When a large number of `INSERT` operations are performed, the data is written into a single Region, causing a write hot spot. To mitigate the hot spot issue, you can configure `SHARD_ROW_ID_BITS`. The row IDs are scattered and the data are written into multiple different Regions. @@ -17,6 +17,26 @@ To mitigate the hot spot issue, you can configure `SHARD_ROW_ID_BITS`. The row I - `SHARD_ROW_ID_BITS = 6` indicates 64 shards - `SHARD_ROW_ID_BITS = 0` indicates the default 1 shard +When you set `SHARD_ROW_ID_BITS = S`, the structure of `_tidb_rowid` is as follows: + +| Sign bit | Shard bits | Auto-increment bits | +|--------|--------|--------------| +| 1 bit | `S` bits | `63-S` bits | + +- The values of the auto-increment bits are stored in TiKV and allocated sequentially. Each time a value is allocated, the next value is incremented by 1. When the value of the auto-increment bits is exhausted (that is, when the maximum value is reached), subsequent automatic allocations fail with the error `Failed to read auto-increment value from storage engine`. +- The value range of `_tidb_rowid`: the maximum number of bits for the final generated value = shard bits + auto-increment bits, so the maximum value is `(2^63)-1`. + +> **Warning:** +> +> `_tidb_rowid` is an internal row ID implicitly assigned by TiDB. Do not assume it is globally unique in all cases. For partitioned tables that do not use clustered indexes, `ALTER TABLE ... EXCHANGE PARTITION` can leave different partitions with the same `_tidb_rowid` value. For details, see [`_tidb_rowid`](https://docs.pingcap.com/tidb/v7.5/tidb-rowid/). + +> **Note:** +> +> Selection of shard bits (`S`): +> +> - Because the total bits of `_tidb_rowid` is 64, the number of shard bits affects the number of auto-increment bits: when the number of shard bits increases, the number of auto-increment bits decreases, and vice versa. Therefore, you need to balance the randomness of the auto-increment values and the available auto-increment space. +> - The best practice is to set the shard bits to `log(2, x)`, where `x` is the number of TiKV nodes in the cluster. For example, if there are 16 TiKV nodes in a TiDB cluster, it is recommended to set the shard bits to `log(2, 16)`, which equals `4`. After all Regions are evenly scheduled to each TiKV node, the load of bulk writes can be evenly distributed to different TiKV nodes to maximize resource utilization. + For details on the usage, see [the Troubleshoot Hotspot Issues guide](/troubleshoot-hot-spot-issues.md#use-shard_row_id_bits-to-process-hotspots). diff --git a/smooth-upgrade-tidb.md b/smooth-upgrade-tidb.md index f73a2a62b1bf1..c004e1652a02a 100644 --- a/smooth-upgrade-tidb.md +++ b/smooth-upgrade-tidb.md @@ -5,39 +5,84 @@ summary: This document introduces the smooth upgrade feature of TiDB, which supp # TiDB Smooth Upgrade -> **Warning:** -> -> Smooth upgrade is still an experimental feature. - This document introduces the smooth upgrade feature of TiDB, which supports upgrading TiDB clusters without manually canceling DDL operations. -Starting from v7.1.0, when you upgrade TiDB to a later version, TiDB supports smooth upgrade. This feature removes the limitations during the upgrade process and provides a more user-friendly upgrade experience. This feature is enabled by default and cannot be disabled. +Starting from v7.1.0, when you upgrade TiDB to a later version, TiDB supports smooth upgrade. This feature removes the limitations during the upgrade process and provides a more user-friendly upgrade experience. Note that you need to ensure that there are no user-initiated DDL operations during the upgrade process. + +## Supported versions + +Depending on whether the feature needs to be controlled by a switch, there are two ways to use smooth upgrade: + +- The feature is enabled by default and does not need to be controlled by a switch. Currently, the versions that support this method are v7.1.0, v7.1.1, v7.2.0, and v7.3.0. The specific supported versions are as follows: + - Upgrade from v7.1.0 to v7.1.1, v7.2.0, or v7.3.0 + - Upgrade from v7.1.1 to v7.2.0 or v7.3.0 + - Upgrade from v7.2.0 to v7.3.0 + +- The feature is disabled by default, and can be enabled by sending the `/upgrade/start` request. For details, see [TiDB HTTP API](https://github.com/pingcap/tidb/blob/master/docs/tidb_http_api.md). The supported versions are as follows: + - Upgrade from v7.1.2 and later v7.1 versions (that is, v7.1.x, where x >= 2) to v7.4.0 and later versions + - Upgrade from v7.4.0 to later versions + +Refer to the following table for the upgrade methods supported by specific versions: + +| Original version | Upgraded version | Upgrade methods | Note | +|------|--------|-------------|-------------| +| < v7.1.0 | Any version | Does not support smooth upgrade. | | +| v7.1.0 | v7.1.1、v7.2.0, or v7.3.0 | Smooth upgrade is automatically supported. No additional operations are required. | Experimental feature. It might encounter the issue [#44760](https://github.com/pingcap/tidb/pull/44760). | +| v7.1.1 | v7.2.0 or v7.3.0 | Smooth upgrade is automatically supported. No additional operations are required. | Experimental feature. | +| v7.2.0 | v7.3.0 | Smooth upgrade is automatically supported. No additional operations are required. | Experimental feature. | +| [v7.1.2, v7.2.0) | [v7.1.2, v7.2.0) | Enable smooth upgrade by sending the `/upgrade/start` HTTP request. There are two methods: [Use TiUP](#use-tiup-to-upgrade) and [Other upgrade methods](#other-upgrade-methods) | When smooth upgrade is not enabled, ensure that no DDL operations are performed during the upgrade. | +| [v7.1.2, v7.2.0) or >= v7.4.0 | >= v7.4.0 | Enable smooth upgrade by sending the `/upgrade/start` HTTP request. There are two methods: [Use TiUP](#use-tiup-to-upgrade) and [Other upgrade methods](#other-upgrade-methods) | When smooth upgrade is not enabled, ensure that no DDL operations are performed during the upgrade. | +| v7.1.0, v7.1.1, v7.2.0, and v7.3.0 | >= v7.4.0 | Does not support smooth upgrade. | | ## Feature introduction -Before the smooth upgrade feature is introduced, there are the following limitations on DDL operations during the upgrade process (see the *Warning* content in [Upgrade TiDB Using TiUP](/upgrade-tidb-using-tiup.md#upgrade-tidb-using-tiup)): +Before the smooth upgrade feature is introduced, there are the following limitations on DDL operations during the upgrade process: - Running DDL operations during the upgrade process might cause undefined behavior in TiDB. - Upgrading TiDB during the DDL operations might cause undefined behavior in TiDB. -After the smooth upgrade feature is introduced, the upgrade process is no longer subject to the preceding limitations. +These limitations can be summarized as that you need to ensure that there are no user-initiated DDL operations during the upgrade process. After the smooth upgrade feature is introduced, TiDB is no longer subject to this limitation during the upgrade process. + +For more information, see the **Warning** content in [Upgrade TiDB Using TiUP](/upgrade-tidb-using-tiup.md#upgrade-tidb-using-tiup). + +### Upgrade steps + +#### Use TiUP to upgrade -During the upgrade process, TiDB automatically performs the following operations without user intervention: +Starting from v1.14.0, TiUP automatically supports this feature. That is, you can directly use the `tiup cluster upgrade` command to upgrade TiDB clusters. Note that the `tiup cluster patch` command is not supported currently. -1. Pause user DDL operations. -2. Perform system DDL operations for the upgrade. -3. Resume the paused user DDL operations. -4. Complete the upgrade. +#### Use TiDB Operator to upgrade -The resumed DDL jobs are still executed in the order before the upgrade. +Currently, this feature is not supported. It will be supported as soon as possible. + +#### Other upgrade methods + +You can take the following steps to upgrade TiDB manually or by using a script: + +1. Send the HTTP upgrade start request to any TiDB node in the cluster: `curl -X POST http://{TiDBIP}:10080/upgrade/start`. + * The TiDB cluster enters the **Upgrading** state. + * The DDL operations to be performed are paused. + +2. Replace the TiDB binary and perform a rolling upgrade. This process is the same as the original upgrade process. + * The system DDL operations are performed during the upgrade process. + +3. After all TiDB nodes in the cluster are upgraded successfully, send the HTTP upgrade finish request to any TiDB node: `curl -X POST http://{TiDBIP}:10080/upgrade/finish`. + * The paused DDL operations of users are resumed. ## Limitations When using the smooth upgrade feature, note the following limitations. +> **Note:** +> +> The limitations in this section apply not only to scenarios using the smooth upgrade feature, but also to [upgrading TiDB using TiUP](/upgrade-tidb-using-tiup.md#upgrade-tidb-using-tiup). + ### Limitations on user operations -* Before the upgrade, if there is a canceling DDL job in the cluster, that is, an ongoing DDL job is being canceled by a user, because the job in the canceling state cannot be paused, TiDB will retry canceling the job. If the retry fails, an error is reported and the upgrade is exited. +* Before the upgrade, consider the following restrictions: + + * If there is a canceling DDL job in the cluster, that is, an ongoing DDL job is being canceled by a user, because the job in the canceling state cannot be paused, TiDB will retry canceling the job. If the retry fails, an error is reported and the upgrade is exited. + * If the TiDB Distributed eXecution Framework (DXF) is enabled, disable it by setting [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) to `OFF` (its default value). Make sure that all ongoing distributed `ADD INDEX` and `IMPORT INTO` tasks are completed. Alternatively, you can cancel these tasks and wait until the upgrade is complete to restart them. Otherwise, the `ADD INDEX` operations during the upgrade might cause data index inconsistency. * In scenarios of using TiUP to upgrade TiDB, because TiUP upgrade has a timeout period, if the cluster has a large number of DDL jobs (more than 300) waiting in queues before the upgrade, the upgrade might fail. @@ -54,3 +99,7 @@ When using the smooth upgrade feature, note the following limitations. * BR: BR might replicate the paused DDL jobs to TiDB. The paused DDL jobs cannot be automatically resumed, which might cause the DDL jobs to be stuck later. * DM and TiCDC: If you use DM or TiCDC to import SQL statements to TiDB during the upgrade process, and if one of the SQL statements contains DDL operations, the import operation is blocked and undefined errors might occur. + +### Limitation on plugins + +The plugins installed in TiDB might contain DDL operations. However, during the upgrade, if the DDL operations in the plugins are performed on non-system tables, the upgrade might fail. diff --git a/sql-logical-optimization.md b/sql-logical-optimization.md index bcb86ca646a4a..0e653ae5883c5 100644 --- a/sql-logical-optimization.md +++ b/sql-logical-optimization.md @@ -1,5 +1,6 @@ --- title: SQL Logical Optimization +summary: SQL Logical Optimization chapter explains key logic rewrites in TiDB query plan generation. For example, `IN` sub-query `t.a in (select t1.a from t1 where t1.b=t.b)` does not exist due to TiDB rewrites. Key rewrites include Subquery Related Optimizations, Column Pruning, Decorrelation of Correlated Subquery, Eliminate Max/Min, Predicates Push Down, Partition Pruning, TopN and Limit Operator Push Down, and Join Reorder. --- # SQL Logical Optimization @@ -15,4 +16,5 @@ This chapter introduces the following key rewrites: - [Predicates Push Down](/predicate-push-down.md) - [Partition Pruning](/partition-pruning.md) - [TopN and Limit Operator Push Down](/topn-limit-push-down.md) -- [Join Reorder](/join-reorder.md) \ No newline at end of file +- [Join Reorder](/join-reorder.md) +- [Derive TopN or Limit from Window Functions](/derive-topn-from-window.md) \ No newline at end of file diff --git a/sql-mode.md b/sql-mode.md index 5b6c4a3fb2989..46f4fa46c3f68 100644 --- a/sql-mode.md +++ b/sql-mode.md @@ -1,18 +1,19 @@ --- title: SQL Mode summary: Learn SQL mode. -aliases: ['/docs/dev/sql-mode/','/docs/dev/reference/sql/sql-mode/'] --- # SQL Mode TiDB servers operate in different SQL modes and apply these modes differently for different clients. SQL mode defines the SQL syntaxes that TiDB supports and the type of data validation check to perform, as described below: -After TiDB is started, modify `SET [ SESSION | GLOBAL ] sql_mode='modes'` to set SQL mode. +After TiDB is started, you can use the `SET [ SESSION | GLOBAL ] sql_mode='modes'` statement to set SQL mode. -Ensure that you have `SUPER` privilege when setting SQL mode at `GLOBAL` level, and your setting at this level only affects the connections established afterwards. Changes to SQL mode at `SESSION` level only affect the current client. +- Ensure that you have `SUPER` privilege when setting SQL mode at `GLOBAL` level, and your setting at this level only affects the connections established afterwards. -`Modes` are a series of different modes separated by commas (','). You can use the `SELECT @@sql_mode` statement to check the current SQL mode. The default value of SQL mode: `ONLY_FULL_GROUP_BY, STRICT_TRANS_TABLES, NO_ZERO_IN_DATE, NO_ZERO_DATE, ERROR_FOR_DIVISION_BY_ZERO, NO_AUTO_CREATE_USER, NO_ENGINE_SUBSTITUTION`. +- Changes to SQL mode at `SESSION` level only affect the current client. + +In this statement, `modes` is a set of modes separated by commas (','). You can use the `SELECT @@sql_mode` statement to check the current SQL mode. The default value of SQL mode: `ONLY_FULL_GROUP_BY, STRICT_TRANS_TABLES, NO_ZERO_IN_DATE, NO_ZERO_DATE, ERROR_FOR_DIVISION_BY_ZERO, NO_AUTO_CREATE_USER, NO_ENGINE_SUBSTITUTION`. ## Important `sql_mode` values @@ -27,7 +28,7 @@ Ensure that you have `SUPER` privilege when setting SQL mode at `GLOBAL` level, | `PIPES_AS_CONCAT` | Treats "\|\|" as a string concatenation operator (`+`) (the same as `CONCAT()`), not as an `OR` (full support) | | `ANSI_QUOTES` | Treats `"` as an identifier. If `ANSI_QUOTES` is enabled, only single quotes are treated as string literals, and double quotes are treated as identifiers. Therefore, double quotes cannot be used to quote strings. (full support)| | `IGNORE_SPACE` | If this mode is enabled, the system ignores space. For example: "user" and "user " are the same. (full support)| -| `ONLY_FULL_GROUP_BY` | If a non-aggregated column that is referred to in `SELECT`, `HAVING`, or `ORDER BY` is absent in `GROUP BY`, this SQL statement is invalid, because it is abnormal for a column to be absent in `GROUP BY` but displayed by query. (full support) | +| `ONLY_FULL_GROUP_BY` | A SQL statement is invalid if it refers to a column in `SELECT`, `HAVING`, or `ORDER BY` that is neither aggregated nor included in the `GROUP BY` clause. This is because displaying such a column in query result is abnormal. (full support)| | `NO_UNSIGNED_SUBTRACTION` | Does not mark the result as `UNSIGNED` if an operand has no symbol in subtraction. (full support)| | `NO_DIR_IN_CREATE` | Ignores all `INDEX DIRECTORY` and `DATA DIRECTORY` directives when a table is created. This option is only useful for secondary replication servers (syntax support only) | | `NO_KEY_OPTIONS` | When you use the `SHOW CREATE TABLE` statement, MySQL-specific syntaxes such as `ENGINE` are not exported. Consider this option when migrating across DB types using mysqldump. (syntax support only)| @@ -52,6 +53,6 @@ Ensure that you have `SUPER` privilege when setting SQL mode at `GLOBAL` level, | `MAXDB` | Equivalent to `PIPES_AS_CONCAT`, `ANSI_QUOTES`, `IGNORE_SPACE`, `NO_KEY_OPTIONS`, `NO_TABLE_OPTIONS`, `NO_FIELD_OPTIONS`, `NO_AUTO_CREATE_USER` (full support)| | `MySQL323` | Equivalent to `NO_FIELD_OPTIONS`, `HIGH_NOT_PRECEDENCE` (syntax support only)| | `MYSQL40` | Equivalent to `NO_FIELD_OPTIONS`, `HIGH_NOT_PRECEDENCE` (syntax support only)| -| `ANSI` | Equivalent to `REAL_AS_FLOAT`, `PIPES_AS_CONCAT`, `ANSI_QUOTES`, `IGNORE_SPACE` (syntax support only)| +| `ANSI` | Equivalent to `REAL_AS_FLOAT`, `PIPES_AS_CONCAT`, `ANSI_QUOTES`, `IGNORE_SPACE`, `ONLY_FULL_GROUP_BY` (syntax support only)| | `TRADITIONAL` | Equivalent to `STRICT_TRANS_TABLES`, `STRICT_ALL_TABLES`, `NO_ZERO_IN_DATE`, `NO_ZERO_DATE`, `ERROR_FOR_DIVISION_BY_ZERO`, `NO_AUTO_CREATE_USER` (syntax support only) | | `ORACLE` | Equivalent to `PIPES_AS_CONCAT`, `ANSI_QUOTES`, `IGNORE_SPACE`, `NO_KEY_OPTIONS`, `NO_TABLE_OPTIONS`, `NO_FIELD_OPTIONS`, `NO_AUTO_CREATE_USER` (syntax support only)| diff --git a/sql-non-prepared-plan-cache.md b/sql-non-prepared-plan-cache.md index 028ad610a25d8..35eb61467fd5b 100644 --- a/sql-non-prepared-plan-cache.md +++ b/sql-non-prepared-plan-cache.md @@ -86,7 +86,7 @@ Due to the preceding risks and the fact that the execution plan cache only provi - Queries that contain numbers or expressions directly after `ORDER BY` or `GROUP BY` are not supported, such as `ORDER BY 1` and `GROUP BY a+1`. Only `ORDER BY column_name` and `GROUP BY column_name` are supported. - Queries that filter on columns of `JSON`, `ENUM`, `SET`, or `BIT` type are not supported, such as `SELECT * FROM t WHERE json_col = '{}'`. - Queries that filter on `NULL` values are not supported, such as `SELECT * FROM t WHERE a is NULL`. -- Queries with more than 200 parameters after parameterization are not supported by default, such as `SELECT * FROM t WHERE a in (1, 2, 3, ... 201)`. Starting from v7.3.0, you can modify this limit by setting the [`44823`](/optimizer-fix-controls.md#44823-new-in-v730) fix in the [`tidb_opt_fix_control`](/system-variables.md#tidb_opt_fix_control-new-in-v710) system variable. +- Queries with more than 200 parameters after parameterization are not supported by default, such as `SELECT * FROM t WHERE a in (1, 2, 3, ... 201)`. Starting from v7.3.0, you can modify this limit by setting the [`44823`](/optimizer-fix-controls.md#44823-new-in-v730) fix in the [`tidb_opt_fix_control`](/system-variables.md#tidb_opt_fix_control-new-in-v653-and-v710) system variable. - Queries that access partitioned tables, virtual columns, temporary tables, views, or memory tables are not supported, such as `SELECT * FROM INFORMATION_SCHEMA.COLUMNS`, where `COLUMNS` is a TiDB memory table. - Queries with hints or bindings are not supported. - DML statements or `SELECT` statements with the `FOR UPDATE` clause are not supported by default. To remove this restriction, you can execute `SET tidb_enable_non_prepared_plan_cache_for_dml = ON`. @@ -112,7 +112,7 @@ Note that if you do not add `FORMAT='plan_cache'`, the `EXPLAIN` statement will To verify whether the query hits the cache, execute the following `EXPLAIN FORMAT='plan_cache'` statement: ```sql -EXPLAIN FORMAT='plan_cache' SELECT * FROM (SELECT a+1 FROM t1) t; +EXPLAIN FORMAT='plan_cache' SELECT * FROM (SELECT a+1 FROM t) t; ``` The output is as follows: diff --git a/sql-optimization-concepts.md b/sql-optimization-concepts.md index e1c8935f02eae..a01b67910f851 100644 --- a/sql-optimization-concepts.md +++ b/sql-optimization-concepts.md @@ -1,7 +1,6 @@ --- title: SQL Optimization Process summary: Learn about the logical and physical optimization of SQL in TiDB. -aliases: ['/docs/dev/sql-optimization-concepts/','/docs/dev/reference/performance/sql-optimizer-overview/'] --- # SQL Optimization Process diff --git a/sql-physical-optimization.md b/sql-physical-optimization.md index 0f52a5e8cbed5..40c0a0cb9bfce 100644 --- a/sql-physical-optimization.md +++ b/sql-physical-optimization.md @@ -1,5 +1,6 @@ --- title: SQL Physical Optimization +summary: Physical optimization is a cost-based process that creates a physical execution plan for the logical execution plan. The optimizer selects the best physical implementation for each operator based on data statistics, time complexity, and resource consumption. This includes index selection, statistics collection, using the right index, distinct keyword optimization, and cost model for optimal execution plan selection. --- # SQL Physical Optimization diff --git a/sql-plan-management.md b/sql-plan-management.md index 6e80d4a6d46f4..86e522101ac31 100644 --- a/sql-plan-management.md +++ b/sql-plan-management.md @@ -1,7 +1,6 @@ --- title: SQL Plan Management (SPM) summary: Learn about SQL Plan Management in TiDB. -aliases: ['/docs/dev/sql-plan-management/','/docs/dev/reference/performance/execution-plan-bind/','/docs/dev/execution-plan-binding/'] --- # SQL Plan Management (SPM) @@ -251,7 +250,7 @@ To use this binding method, you need to first get the `plan_digest` correspondin ```sql CREATE TABLE t(id INT PRIMARY KEY , a INT, KEY(a)); SELECT /*+ IGNORE_INDEX(t, a) */ * FROM t WHERE a = 1; - SELECT * FROM INFORMATION_SCHEMA.STATEMENTS_SUMMARY WHERE QUERY_SAMPLE_TEXT = 'SELECT /*+ IGNORE_INDEX(t, a) */ * FROM t WHERE a = 1'\G; + SELECT * FROM INFORMATION_SCHEMA.STATEMENTS_SUMMARY WHERE QUERY_SAMPLE_TEXT = 'SELECT /*+ IGNORE_INDEX(t, a) */ * FROM t WHERE a = 1'\G ``` The following is a part of the example query result of `statements_summary`: @@ -280,7 +279,7 @@ To use this binding method, you need to first get the `plan_digest` correspondin To verify whether the created binding takes effect, you can [view bindings](#view-bindings): ```sql -SHOW BINDINGS\G; +SHOW BINDINGS\G ``` ``` diff --git a/sql-plan-replayer.md b/sql-plan-replayer.md index d849aafc9e3e3..a4c2ce17d862e 100644 --- a/sql-plan-replayer.md +++ b/sql-plan-replayer.md @@ -16,10 +16,8 @@ The features of `PLAN REPLAYER` are as follows: You can use `PLAN REPLAYER` to save the on-site information of a TiDB cluster. The export interface is as follows: -{{< copyable "sql" >}} - ```sql -PLAN REPLAYER DUMP EXPLAIN [ANALYZE] [WITH STATS AS OF TIMESTAMP expression] sql-statement; +PLAN REPLAYER DUMP [WITH STATS AS OF TIMESTAMP expression] EXPLAIN [ANALYZE] sql-statement; ``` Based on `sql-statement`, TiDB sorts out and exports the following on-site information: @@ -31,7 +29,7 @@ Based on `sql-statement`, TiDB sorts out and exports the following on-site infor - The table schema in `sql-statement` - The statistics of the table in `sql-statement` - The result of `EXPLAIN [ANALYZE] sql-statement` -- Some internal procudures of query optimization +- Some internal procedures of query optimization If historical statistics are [enabled](/system-variables.md#tidb_enable_historical_stats), you can specify a time in the `PLAN REPLAYER` statement to get the historical statistics for the corresponding time. You can directly specify a time and date or specify a timestamp. TiDB looks for the historical statistics before the specified time and exports the latest one among them. @@ -143,7 +141,7 @@ With an existing `ZIP` file exported using `PLAN REPLAYER`, you can use the `PLA PLAN REPLAYER LOAD 'file_name'; ``` -In the statement above, `file_name` is the name of the `ZIP` file to be exported. +In the statement above, `file_name` is the name of the `ZIP` file to be imported. For example: @@ -153,6 +151,16 @@ For example: PLAN REPLAYER LOAD 'plan_replayer.zip'; ``` +> **Note:** +> +> You need to disable auto analyze. Otherwise the imported statistics will be overwritten by analyze. + +You can disable auto analyze by setting the [`tidb_enable_auto_analyze`](/system-variables.md#tidb_enable_auto_analyze-new-in-v610) system variable to `OFF`: + +```sql +set @@global.tidb_enable_auto_analyze = OFF; +``` + After the cluster information is imported, the TiDB cluster is loaded with the required table schema, statistics and other information that affects the construction of the execution plan. You can view the execution plan and verify statistics in the following way: ```sql @@ -221,7 +229,7 @@ PLAN REPLAYER CAPTURE 'sql_digest' '*'; You can view the ongoing capture tasks of `PLAN REPLAYER CAPTURE` in the TiDB cluster using the following statement: ```sql -mysql> PLAN PLAYER CAPTURE 'example_sql' 'example_plan'; +mysql> PLAN REPLAYER CAPTURE 'example_sql' 'example_plan'; Query OK, 1 row affected (0.01 sec) mysql> SELECT * FROM mysql.plan_replayer_task; @@ -255,6 +263,29 @@ The method of downloading the file of `PLAN REPLAYER CAPTURE` is the same as tha > > The result file of `PLAN REPLAYER CAPTURE` is kept in the TiDB cluster for up to one week. After one week, TiDB deletes the file. +### Remove the capture tasks + +If a capture task is no longer needed, you can remove it using the `PLAN REPLAYER CAPTURE REMOVE` statement. For example: + +```sql +mysql> PLAN REPLAYER CAPTURE '077a87a576e42360c95530ccdac7a1771c4efba17619e26be50a4cfd967204a0' '4838af52c1e07fc8694761ad193d16a689b2128bc5ced9d13beb31ae27b370ce'; +Query OK, 0 rows affected (0.01 sec) + +mysql> SELECT * FROM mysql.plan_replayer_task; ++------------------------------------------------------------------+------------------------------------------------------------------+---------------------+ +| sql_digest | plan_digest | update_time | ++------------------------------------------------------------------+------------------------------------------------------------------+---------------------+ +| 077a87a576e42360c95530ccdac7a1771c4efba17619e26be50a4cfd967204a0 | 4838af52c1e07fc8694761ad193d16a689b2128bc5ced9d13beb31ae27b370ce | 2024-05-21 11:26:10 | ++------------------------------------------------------------------+------------------------------------------------------------------+---------------------+ +1 row in set (0.01 sec) + +mysql> PLAN REPLAYER CAPTURE REMOVE '077a87a576e42360c95530ccdac7a1771c4efba17619e26be50a4cfd967204a0' '4838af52c1e07fc8694761ad193d16a689b2128bc5ced9d13beb31ae27b370ce'; +Query OK, 0 rows affected (0.01 sec) + +mysql> SELECT * FROM mysql.plan_replayer_task; +Empty set (0.01 sec) +``` + ## Use `PLAN REPLAYER CONTINUOUS CAPTURE` After `PLAN REPLAYER CONTINUOUS CAPTURE` is enabled, TiDB asynchronously records the applications' SQL statements with the `PLAN REPLAYER` method according to their `SQL DIGEST` and `PLAN DIGEST`. For SQL statements and execution plans that share the same DIGEST, `PLAN REPLAYER CONTINUOUS CAPTURE` does not record them repeatedly. diff --git a/sql-prepared-plan-cache.md b/sql-prepared-plan-cache.md index 2c3bb789993f3..4e15d77516dc3 100644 --- a/sql-prepared-plan-cache.md +++ b/sql-prepared-plan-cache.md @@ -1,11 +1,17 @@ --- title: SQL Prepared Execution Plan Cache summary: Learn about SQL Prepared Execution Plan Cache in TiDB. -aliases: ['/tidb/dev/sql-prepare-plan-cache'] --- # SQL Prepared Execution Plan Cache +> **Warning:** +> +> If a cached `UPDATE` or `DELETE` statement encounters a DDL operation that modifies the relevant schema during execution, it might cause data inconsistency between tables and indexes. For more information, see [Issue #51407](https://github.com/pingcap/tidb/issues/51407). It is recommended to monitor the status of this issue and upgrade to [the latest LTS version](https://docs.pingcap.com/tidb/stable) to resolve this issue. Before upgrading, you can try the following workarounds: +> +> - Before executing DDL, temporarily [disable the prepared plan cache](/system-variables.md#tidb_enable_prepared_plan_cache-new-in-v610), and re-enable it after the DDL is complete. +> - Avoid executing DDL during business peak hours. After executing DDL, immediately run [`ADMIN CHECK TABLE`](/sql-statements/sql-statement-admin-check-table-index.md) to check the consistency between tables and indexes. If errors are found, rebuild the relevant indexes. + TiDB supports execution plan caching for `Prepare` and `Execute` queries. This includes both forms of prepared statements: - Using the `COM_STMT_PREPARE` and `COM_STMT_EXECUTE` protocol features. diff --git a/sql-statements/sql-statement-add-column.md b/sql-statements/sql-statement-add-column.md index 47ae0faa329e1..1863fd73ed7cf 100644 --- a/sql-statements/sql-statement-add-column.md +++ b/sql-statements/sql-statement-add-column.md @@ -1,7 +1,6 @@ --- title: ADD COLUMN | TiDB SQL Statement Reference summary: An overview of the usage of ADD COLUMN for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-add-column/','/docs/dev/reference/sql/statements/add-column/'] --- # ADD COLUMN diff --git a/sql-statements/sql-statement-add-index.md b/sql-statements/sql-statement-add-index.md index c85653df8f98f..d770670aa6600 100644 --- a/sql-statements/sql-statement-add-index.md +++ b/sql-statements/sql-statement-add-index.md @@ -1,13 +1,20 @@ --- title: ADD INDEX | TiDB SQL Statement Reference summary: An overview of the usage of ADD INDEX for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-add-index/','/docs/dev/reference/sql/statements/add-index/'] --- # ADD INDEX The `ALTER TABLE.. ADD INDEX` statement adds an index to an existing table. This operation is online in TiDB, which means that neither reads or writes to the table are blocked by adding an index. + + +> **Note:** +> +> For [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters with 4 vCPU, it is recommended to manually disable [`tidb_ddl_enable_fast_reorg`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) to prevent resource limitations from affecting cluster stability during index creation. Disabling this setting allows indexes to be created using transactions, which reduces the overall impact on the cluster. + + + > **Warning:** diff --git a/sql-statements/sql-statement-admin-cancel-ddl.md b/sql-statements/sql-statement-admin-cancel-ddl.md index fe95d83092080..968b059a8785a 100644 --- a/sql-statements/sql-statement-admin-cancel-ddl.md +++ b/sql-statements/sql-statement-admin-cancel-ddl.md @@ -34,7 +34,7 @@ If the operation fails to cancel the jobs, specific reasons are displayed. > **Note:** > -> - Only this operation can cancel DDL jobs. All other operations and environment changes (such as machine restart and cluster restart) cannot cancel these jobs. +> - Before v6.2.0, only this operation can cancel DDL jobs, and all other operations and environment changes (such as machine restart and cluster restart) cannot cancel these jobs. Starting from v6.2.0, [`KILL`](/sql-statements/sql-statement-kill.md) statements can also be used to cancel ongoing DDL jobs by killing them. > - This operation can cancel multiple DDL jobs at the same time. You can get the ID of DDL jobs using the [`ADMIN SHOW DDL JOBS`](/sql-statements/sql-statement-admin-show-ddl.md) statement. > - If the jobs you want to cancel are finished, the cancellation operation fails. diff --git a/sql-statements/sql-statement-admin-cleanup.md b/sql-statements/sql-statement-admin-cleanup.md index f10b57705f273..a750ceee4f967 100644 --- a/sql-statements/sql-statement-admin-cleanup.md +++ b/sql-statements/sql-statement-admin-cleanup.md @@ -53,6 +53,23 @@ ADMIN CHECK INDEX tbl idx; Query OK, 0 rows affected (0.01 sec) ``` + + +> **Note:** +> +> When the data and index are inconsistent due to the loss of replicas: +> +> - There might be a loss of both row data and index data. To restore the consistency, use the `ADMIN CLEANUP INDEX` and [`ADMIN RECOVER INDEX`](/sql-statements/sql-statement-admin-recover.md) statements together. +> - The `ADMIN CLEANUP INDEX` statement is always executed in a single thread. When the table data is large, it is recommended to recover the index data by rebuilding the index. +> - When you execute the `ADMIN CLEANUP INDEX` statement, the corresponding table or index is not locked and TiDB allows other sessions to modify the table records at the same time. However, in this case, `ADMIN CLEANUP INDEX` might not be able to handle all table records correctly. Therefore, when you execute `ADMIN CLEANUP INDEX`, avoid modifying the table data at the same time. +> - If you use the enterprise edition of TiDB, you can [submit a request](/support.md) to contact the support engineer for help. +> +> The `ADMIN CLEANUP INDEX` statement is not atomic: if the statement is interrupted during execution, it is recommended to execute it again until it succeeds. + + + + + > **Note:** > > When the data and index are inconsistent due to the loss of replicas: @@ -64,6 +81,8 @@ Query OK, 0 rows affected (0.01 sec) > > The `ADMIN CLEANUP INDEX` statement is not atomic: if the statement is interrupted during execution, it is recommended to execute it again until it succeeds. + + ## MySQL compatibility This statement is a TiDB extension to MySQL syntax. diff --git a/sql-statements/sql-statement-admin-recover.md b/sql-statements/sql-statement-admin-recover.md index 07d8fb6c69fd4..1bcee6c87b933 100644 --- a/sql-statements/sql-statement-admin-recover.md +++ b/sql-statements/sql-statement-admin-recover.md @@ -51,6 +51,23 @@ ADMIN CHECK INDEX tbl idx; Query OK, 0 rows affected (0.01 sec) ``` + + +> **Note:** +> +> When the data and index are inconsistent due to the loss of replicas: +> +> - There might be a loss of both row data and index data. To address the issue, use the [`ADMIN CLEANUP INDEX`](/sql-statements/sql-statement-admin-cleanup.md) and `ADMIN RECOVER INDEX` statements together to recover the consistency of row data and index data. +> - The `ADMIN RECOVER INDEX` statement is always executed in a single thread. When the table data is large, it is recommended to recover the index data by rebuilding the index. +> - When you execute the `ADMIN RECOVER INDEX` statement, the corresponding table or index is not locked and TiDB allows other sessions to modify the table records at the same time. However, in this case, `ADMIN RECOVER INDEX` might not be able to handle all table records correctly. Therefore, when you execute `ADMIN RECOVER INDEX`, avoid modifying the table data at the same time. +> - If you use the enterprise edition of TiDB, you can [submit a request](/support.md) to contact the support engineer for help. +> +> The `ADMIN RECOVER INDEX` statement is not atomic: if the statement is interrupted during execution, it is recommended to execute it again until it succeeds. + + + + + > **Note:** > > When the data and index are inconsistent due to the loss of replicas: @@ -62,6 +79,8 @@ Query OK, 0 rows affected (0.01 sec) > > The `ADMIN RECOVER INDEX` statement is not atomic: if the statement is interrupted during execution, it is recommended to execute it again until it succeeds. + + ## MySQL compatibility This statement is a TiDB extension to MySQL syntax. diff --git a/sql-statements/sql-statement-admin-show-ddl.md b/sql-statements/sql-statement-admin-show-ddl.md index 9da7821eac9b2..f89a837be0bce 100644 --- a/sql-statements/sql-statement-admin-show-ddl.md +++ b/sql-statements/sql-statement-admin-show-ddl.md @@ -10,8 +10,13 @@ The `ADMIN SHOW DDL [JOBS|JOB QUERIES]` statement shows information about runnin ## Synopsis ```ebnf+diagram -AdminStmt ::= - 'ADMIN' ( 'SHOW' ( 'DDL' ( 'JOBS' Int64Num? WhereClauseOptional | 'JOB' 'QUERIES' NumList | 'JOB' 'QUERIES' 'LIMIT' m 'OFFSET' n )? | TableName 'NEXT_ROW_ID' | 'SLOW' AdminShowSlow ) | 'CHECK' ( 'TABLE' TableNameList | 'INDEX' TableName Identifier ( HandleRange ( ',' HandleRange )* )? ) | 'RECOVER' 'INDEX' TableName Identifier | 'CLEANUP' ( 'INDEX' TableName Identifier | 'TABLE' 'LOCK' TableNameList ) | 'CHECKSUM' 'TABLE' TableNameList | 'CANCEL' 'DDL' 'JOBS' NumList | 'RELOAD' ( 'EXPR_PUSHDOWN_BLACKLIST' | 'OPT_RULE_BLACKLIST' | 'BINDINGS' ) | 'PLUGINS' ( 'ENABLE' | 'DISABLE' ) PluginNameList | 'REPAIR' 'TABLE' TableName CreateTableStmt | ( 'FLUSH' | 'CAPTURE' | 'EVOLVE' ) 'BINDINGS' ) +AdminShowDDLStmt ::= + 'ADMIN' 'SHOW' 'DDL' + ( + 'JOBS' Int64Num? WhereClauseOptional + | 'JOB' 'QUERIES' NumList + | 'JOB' 'QUERIES' 'LIMIT' m ( ('OFFSET' | ',') n )? + )? NumList ::= Int64Num ( ',' Int64Num )* @@ -26,7 +31,12 @@ WhereClauseOptional ::= To view the status of the currently running DDL jobs, use `ADMIN SHOW DDL`. The output includes the current schema version, the DDL ID and address of the owner, the running DDL jobs and SQL statements, and the DDL ID of the current TiDB instance. -{{< copyable "sql" >}} +- `SCHEMA_VER`: a number indicating the version of the schema. +- `OWNER_ID`: the UUID of the DDL owner. See also [`TIDB_IS_DDL_OWNER()`](/functions-and-operators/tidb-functions.md). +- `OWNER_ADDRESS`: the IP address of the DDL owner. +- `RUNNING_JOBS`: details about the running DDL job. +- `SELF_ID`: the UUID of the TiDB node to which you are currently connected. If `SELF_ID` is the same as the `OWNER_ID`, it means that you are connected to the DDL owner. +- `QUERY`: the statements of the queries. ```sql ADMIN SHOW DDL; @@ -57,7 +67,7 @@ The `ADMIN SHOW DDL JOBS` statement is used to view all the results in the curre - `txn-merge`: Transactional backfill with a temporary index that gets merged with the original index when the backfill is finished. - `SCHEMA_STATE`: the current state of the schema object that the DDL operates on. If `JOB_TYPE` is `ADD INDEX`, it is the state of the index; if `JOB_TYPE` is `ADD COLUMN`, it is the state of the column; if `JOB_TYPE` is `CREATE TABLE`, it is the state of the table. Common states include the following: - `none`: indicates that it does not exist. Generally, after the `DROP` operation or after the `CREATE` operation fails and rolls back, it will become the `none` state. - - `delete only`, `write only`, `delete reorganization`, `write reorganization`: these four states are intermediate states. For their specific meanings, see [How the Online DDL Asynchronous Change Works in TiDB](/ddl-introduction.md#how-the-online-ddl-asynchronous-change-works-in-tidb). As the intermediate state conversion is fast, these states are generally not visible during operation. Only when performing `ADD INDEX` operation can the `write reorganization` state be seen, indicating that index data is being added. + - `delete only`, `write only`, `delete reorganization`, `write reorganization`: these four states are intermediate states. For their specific meanings, see [How the Online DDL Asynchronous Change Works in TiDB](/best-practices/ddl-introduction.md#how-the-online-ddl-asynchronous-change-works-in-tidb). As the intermediate state conversion is fast, these states are generally not visible during operation. Only when performing `ADD INDEX` operation can the `write reorganization` state be seen, indicating that index data is being added. - `public`: indicates that it exists and is available to users. Generally, after `CREATE TABLE` and `ADD INDEX` (or `ADD COLUMN`) operations are completed, it will become the `public` state, indicating that the newly created table, column, and index can be read and written normally. - `SCHEMA_ID`: the ID of the database where the DDL operation is performed. - `TABLE_ID`: the ID of the table where the DDL operation is performed. diff --git a/sql-statements/sql-statement-admin-show-telemetry.md b/sql-statements/sql-statement-admin-show-telemetry.md index bc2d4c1b08c0b..a3576f511f6bd 100644 --- a/sql-statements/sql-statement-admin-show-telemetry.md +++ b/sql-statements/sql-statement-admin-show-telemetry.md @@ -9,7 +9,7 @@ The `ADMIN SHOW TELEMETRY` statement shows the information that will be reported > **Note:** > -> This feature is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This feature is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). ## Synopsis diff --git a/sql-statements/sql-statement-admin.md b/sql-statements/sql-statement-admin.md index 155f4caf5637d..8b9e3bb5d335c 100644 --- a/sql-statements/sql-statement-admin.md +++ b/sql-statements/sql-statement-admin.md @@ -1,7 +1,6 @@ --- title: ADMIN | TiDB SQL Statement Reference summary: An overview of the usage of ADMIN for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-admin/','/docs/dev/reference/sql/statements/admin/'] --- # ADMIN @@ -64,7 +63,7 @@ The above statement is used to reload the blocklist of logic optimization rules. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ```sql ADMIN PLUGINS ENABLE plugin_name [, plugin_name] ...; @@ -148,7 +147,7 @@ The above statement is used to view the details of some special columns of a tab > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ```sql ADMIN SHOW SLOW RECENT N; diff --git a/sql-statements/sql-statement-alter-database.md b/sql-statements/sql-statement-alter-database.md index c58b33a01c08f..d65ed2df8e1a7 100644 --- a/sql-statements/sql-statement-alter-database.md +++ b/sql-statements/sql-statement-alter-database.md @@ -1,7 +1,6 @@ --- title: ALTER DATABASE | TiDB SQL Statement Reference summary: An overview of the usage of ALTER DATABASE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-alter-database/','/docs/dev/reference/sql/statements/alter-database/'] --- # ALTER DATABASE diff --git a/sql-statements/sql-statement-alter-index.md b/sql-statements/sql-statement-alter-index.md index 3224a68dfaa8a..957c151e89178 100644 --- a/sql-statements/sql-statement-alter-index.md +++ b/sql-statements/sql-statement-alter-index.md @@ -1,7 +1,6 @@ --- title: ALTER INDEX summary: An overview of the usage of ALTER INDEX for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-alter-index/'] --- # ALTER INDEX @@ -118,7 +117,7 @@ Query OK, 0 rows affected (0.02 sec) ## MySQL compatibility * Invisible indexes in TiDB are modeled on the equivalent feature from MySQL 8.0. -* Similiar to MySQL, TiDB does not permit `PRIMARY KEY` indexes to be made invisible. +* Similar to MySQL, TiDB does not permit `PRIMARY KEY` indexes to be made invisible. * MySQL provides an optimizer switch `use_invisible_indexes=on` to make all invisible indexes _visible_ again. This functionality is not available in TiDB. ## See also diff --git a/sql-statements/sql-statement-alter-instance.md b/sql-statements/sql-statement-alter-instance.md index c4a5a776ea919..a4090b1c78b9a 100644 --- a/sql-statements/sql-statement-alter-instance.md +++ b/sql-statements/sql-statement-alter-instance.md @@ -1,7 +1,6 @@ --- title: ALTER INSTANCE summary: Learn the overview of the `ALTER INSTANCE` usage in TiDB. -aliases: ['/docs/dev/sql-statements/sql-statement-alter-instance/','/docs/dev/reference/sql/statements/alter-instance/'] --- # ALTER INSTANCE @@ -10,7 +9,7 @@ The `ALTER INSTANCE` statement is used to make changes to a single TiDB instance > **Note:** > -> [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) can automatically refresh the TLS certificate, so this feature is not applicable to [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) can automatically refresh the TLS certificate, so this feature is not applicable to [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ## RELOAD TLS diff --git a/sql-statements/sql-statement-alter-placement-policy.md b/sql-statements/sql-statement-alter-placement-policy.md index e56848ff57e30..f469eb31677c9 100644 --- a/sql-statements/sql-statement-alter-placement-policy.md +++ b/sql-statements/sql-statement-alter-placement-policy.md @@ -9,7 +9,7 @@ summary: The usage of ALTER PLACEMENT POLICY in TiDB. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. `ALTER PLACEMENT POLICY` _replaces_ the previous policy with the new definition. It does not _merge_ the old policy with the new one. In the following example, `FOLLOWERS=4` is lost when the `ALTER PLACEMENT POLICY` is executed: @@ -68,7 +68,7 @@ AdvancedPlacementOption ::= CREATE PLACEMENT POLICY p1 PRIMARY_REGION="us-east-1" REGIONS="us-east-1,us-west-1"; CREATE TABLE t1 (i INT) PLACEMENT POLICY=p1; -- Assign policy p1 to table t1 ALTER PLACEMENT POLICY p1 PRIMARY_REGION="us-east-1" REGIONS="us-east-1,us-west-1,us-west-2" FOLLOWERS=4; -- The rules of t1 will be updated automatically. -SHOW CREATE PLACEMENT POLICY p1\G; +SHOW CREATE PLACEMENT POLICY p1\G ``` ``` diff --git a/sql-statements/sql-statement-alter-range.md b/sql-statements/sql-statement-alter-range.md index bcf11b636a14b..f8cb02c14a0bb 100644 --- a/sql-statements/sql-statement-alter-range.md +++ b/sql-statements/sql-statement-alter-range.md @@ -7,6 +7,10 @@ summary: An overview of the usage of ALTER RANGE for TiDB. Currently, the `ALTER RANGE` statement can only be used to modify the range of a specific placement policy in TiDB. +> **Note:** +> +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. + ## Synopsis ```ebnf+diagram diff --git a/sql-statements/sql-statement-alter-resource-group.md b/sql-statements/sql-statement-alter-resource-group.md index 25fb54aefdf79..86988c00d7df3 100644 --- a/sql-statements/sql-statement-alter-resource-group.md +++ b/sql-statements/sql-statement-alter-resource-group.md @@ -9,7 +9,7 @@ The `ALTER RESOURCE GROUP` statement is used to modify a resource group in a dat > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-alter-sequence.md b/sql-statements/sql-statement-alter-sequence.md new file mode 100644 index 0000000000000..72f896d467876 --- /dev/null +++ b/sql-statements/sql-statement-alter-sequence.md @@ -0,0 +1,215 @@ +--- +title: ALTER SEQUENCE +summary: An overview of the usage of ALTER SEQUENCE for the TiDB database. +--- + +# ALTER SEQUENCE + +The `ALTER SEQUENCE` statement alters sequence objects in TiDB. The sequence is a database object that is on a par with the `Table` and the `View` object. The sequence is used to generate serialized IDs in a customized way. + +## Synopsis + +```ebnf+diagram +CreateSequenceStmt ::= + 'ALTER' 'SEQUENCE' TableName CreateSequenceOptionListOpt + +TableName ::= + Identifier ('.' Identifier)? + +CreateSequenceOptionListOpt ::= + SequenceOption* + +SequenceOptionList ::= + SequenceOption + +SequenceOption ::= + ( 'INCREMENT' ( '='? | 'BY' ) | 'START' ( '='? | 'WITH' ) | ( 'MINVALUE' | 'MAXVALUE' | 'CACHE' ) '='? ) SignedNum +| 'COMMENT' '='? stringLit +| 'NOMINVALUE' +| 'NO' ( 'MINVALUE' | 'MAXVALUE' | 'CACHE' | 'CYCLE' ) +| 'NOMAXVALUE' +| 'NOCACHE' +| 'CYCLE' +| 'NOCYCLE' +| 'RESTART' ( ( '='? | 'WITH' ) SignedNum )? +``` + +## Syntax + +```sql +ALTER SEQUENCE sequence_name + [ INCREMENT [ BY | = ] increment ] + [ MINVALUE [=] minvalue | NO MINVALUE | NOMINVALUE ] + [ MAXVALUE [=] maxvalue | NO MAXVALUE | NOMAXVALUE ] + [ START [ WITH | = ] start ] + [ CACHE [=] cache | NOCACHE | NO CACHE] + [ CYCLE | NOCYCLE | NO CYCLE] + [table_options] +``` + +## Parameters + +|Parameters | Default value | Description | +| :-- | :-- | :--| +| `INCREMENT` | `1` | Specifies the increment of a sequence. Its positive or negative values can control the growth direction of the sequence. | +| `MINVALUE` | `1` or `-9223372036854775807` | Specifies the minimum value of a sequence. When `INCREMENT` > `0`, the default value is `1`. When `INCREMENT` < `0`, the default value is `-9223372036854775807`. | +| `MAXVALUE` | `9223372036854775806` or `-1` | Specifies the maximum value of a sequence. When `INCREMENT` > `0`, the default value is `9223372036854775806`. When `INCREMENT` < `0`, the default value is `-1`. | +| `START` | `MINVALUE` or `MAXVALUE`| Specifies the initial value of a sequence. When `INCREMENT` > `0`, the default value is `MINVALUE`. When `INCREMENT` < `0`, the default value is `MAXVALUE`. | +| `CACHE` | `1000` | Specifies the local cache size of a sequence in TiDB. | +| `CYCLE` | `NO CYCLE` | Specifies whether a sequence restarts from the minimum value (or maximum for the descending sequence). When `INCREMENT` > `0`, the default value is `MINVALUE`. When `INCREMENT` < `0`, the default value is `MAXVALUE`. | + +> **Note:** +> +> Changing the `START` value does not affect the generated values until you execute `ALTER SEQUENCE ... RESTART`. + +## `SEQUENCE` function + +You can control a sequence through the following expression functions: + ++ `NEXTVAL` or `NEXT VALUE FOR` + + Essentially, both are the `NEXTVAL()` function that gets the next valid value of a sequence object. The parameter of the `NEXTVAL()` function is the `identifier` of the sequence. + ++ `LASTVAL` + + This function gets the last used value of this session. If the value does not exist, `NULL` is used. The parameter of this function is the `identifier` of the sequence. + ++ `SETVAL` + + This function sets the progression of the current value for a sequence. The first parameter of this function is the `identifier` of the sequence; the second parameter is `num`. + +> **Note:** +> +> In the implementation of a sequence in TiDB, the `SETVAL` function cannot change the initial progression or cycle progression of this sequence. This function only returns the next valid value based on this progression. + +## Examples + +Create a sequence named `s1`: + +```sql +CREATE SEQUENCE s1; +``` + +``` +Query OK, 0 rows affected (0.15 sec) +``` + +Get the next two values from the sequence by executing the following SQL statement twice: + +```sql +SELECT NEXTVAL(s1); +``` + +``` ++-------------+ +| NEXTVAL(s1) | ++-------------+ +| 1 | ++-------------+ +1 row in set (0.01 sec) +``` + +```sql +SELECT NEXTVAL(s1); +``` + +``` ++-------------+ +| NEXTVAL(s1) | ++-------------+ +| 2 | ++-------------+ +1 row in set (0.00 sec) +``` + +Change the increment of the sequence to `2`: + +```sql +ALTER SEQUENCE s1 INCREMENT=2; +``` + +``` +Query OK, 0 rows affected (0.18 sec) +``` + +Now, get the next two values from the sequence again: + +```sql +SELECT NEXTVAL(s1); +``` + +``` ++-------------+ +| NEXTVAL(s1) | ++-------------+ +| 1001 | ++-------------+ +1 row in set (0.02 sec) +``` + +```sql +SELECT NEXTVAL(s1); +``` + +``` ++-------------+ +| NEXTVAL(s1) | ++-------------+ +| 1003 | ++-------------+ +1 row in set (0.00 sec) +``` + +As you can see from the output, the values now increase by two, following the `ALTER SEQUENCE` statement. + +You can also change other parameters of the sequence. For example, you can change the `MAXVALUE` of the sequence as follows: + +```sql +CREATE SEQUENCE s2 MAXVALUE=10; +``` + +``` +Query OK, 0 rows affected (0.17 sec) +``` + +```sql +ALTER SEQUENCE s2 MAXVALUE=100; +``` + +``` +Query OK, 0 rows affected (0.15 sec) +``` + +```sql +SHOW CREATE SEQUENCE s2\G +``` + +``` +*************************** 1. row *************************** + Sequence: s2 +Create Sequence: CREATE SEQUENCE `s2` start with 1 minvalue 1 maxvalue 100 increment by 1 cache 1000 nocycle ENGINE=InnoDB +1 row in set (0.00 sec) +``` + +## MySQL compatibility + +This statement is a TiDB extension. The implementation is modeled on sequences available in MariaDB. + +Except for the `SETVAL` function, all other functions have the same _progressions_ as MariaDB. Here "progression" means that the numbers in a sequence follow a certain arithmetic progression rule defined by the sequence. Although you can use `SETVAL` to set the current value of a sequence, the subsequent values of the sequence still follow the original progression rule. + +For example: + +``` +1, 3, 5, ... // The sequence starts from 1 and increments by 2. +SELECT SETVAL(seq, 6) // Sets the current value of a sequence to 6. +7, 9, 11, ... // Subsequent values still follow the progression rule. +``` + +In the `CYCLE` mode, the initial value of a sequence in the first round is the value of the `START` parameter, and the initial value in the subsequent rounds is the value of `MinValue` (`INCREMENT` > 0) or `MaxValue` (`INCREMENT` < 0). + +## See also + +* [CREATE SEQUENCE](/sql-statements/sql-statement-create-sequence.md) +* [DROP SEQUENCE](/sql-statements/sql-statement-drop-sequence.md) +* [SHOW CREATE SEQUENCE](/sql-statements/sql-statement-show-create-sequence.md) +* [Sequence Functions](/functions-and-operators/sequence-functions.md) diff --git a/sql-statements/sql-statement-alter-table.md b/sql-statements/sql-statement-alter-table.md index a3b28c54cc46c..854deaf1c1c0f 100644 --- a/sql-statements/sql-statement-alter-table.md +++ b/sql-statements/sql-statement-alter-table.md @@ -1,7 +1,6 @@ --- title: ALTER TABLE | TiDB SQL Statement Reference summary: An overview of the usage of ALTER TABLE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-alter-table/','/docs/dev/reference/sql/statements/alter-table/'] --- # ALTER TABLE @@ -120,7 +119,7 @@ Query OK, 0 rows affected (0.30 sec) 2 rows in set (0.00 sec) ``` -TiDB supports the ability to assert that DDL changes will use a particular `ALTER` algorithm. This is only an assertion, and does not change the actual algorithm which will be used to modify the table. It can be useful if you only want to permit instant DDL changes during the peak hours of your cluster: +TiDB supports the ability to assert that DDL changes use a particular `ALTER` algorithm. Note that this is only an assertion, and does not change the actual algorithm used to modify the table: {{< copyable "sql" >}} diff --git a/sql-statements/sql-statement-alter-user.md b/sql-statements/sql-statement-alter-user.md index 62cacea484759..c3ce7145c0350 100644 --- a/sql-statements/sql-statement-alter-user.md +++ b/sql-statements/sql-statement-alter-user.md @@ -1,7 +1,6 @@ --- title: ALTER USER | TiDB SQL Statement Reference summary: An overview of the usage of ALTER USER for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-alter-user/','/docs/dev/reference/sql/statements/alter-user/'] --- # ALTER USER @@ -20,6 +19,12 @@ UserSpecList ::= UserSpec ::= Username AuthOption +RequireClauseOpt ::= + ( 'REQUIRE' 'NONE' | 'REQUIRE' 'SSL' | 'REQUIRE' 'X509' | 'REQUIRE' RequireList )? + +RequireList ::= + ( "ISSUER" stringLit | "SUBJECT" stringLit | "CIPHER" stringLit | "SAN" stringLit | "TOKEN_ISSUER" stringLit )* + Username ::= StringName ('@' StringName | singleAtIdentifier)? | 'CURRENT_USER' OptionalBraces @@ -33,6 +38,10 @@ LockOption ::= ( 'ACCOUNT' 'LOCK' | 'ACCOUNT' 'UNLOCK' )? AttributeOption ::= ( 'COMMENT' CommentString | 'ATTRIBUTE' AttributeString )? ResourceGroupNameOption::= ( 'RESOURCE' 'GROUP' Identifier)? + +RequireClauseOpt ::= ('REQUIRE' ('NONE' | 'SSL' | 'X509' | RequireListElement ('AND'? RequireListElement)*))? + +RequireListElement ::= 'ISSUER' Issuer | 'SUBJECT' Subject | 'CIPHER' Cipher | 'SAN' SAN | 'TOKEN_ISSUER' TokenIssuer ``` ## Examples diff --git a/sql-statements/sql-statement-analyze-table.md b/sql-statements/sql-statement-analyze-table.md index a26f6d05ed40a..b277481c3f634 100644 --- a/sql-statements/sql-statement-analyze-table.md +++ b/sql-statements/sql-statement-analyze-table.md @@ -1,7 +1,6 @@ --- title: ANALYZE | TiDB SQL Statement Reference summary: An overview of the usage of ANALYZE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-analyze-table/','/docs/dev/reference/sql/statements/analyze-table/'] --- # ANALYZE @@ -16,7 +15,7 @@ Currently, TiDB collects statistical information as a full collection by using t ```ebnf+diagram AnalyzeTableStmt ::= - 'ANALYZE' ( 'TABLE' ( TableNameList ( 'ALL COLUMNS' | 'PREDICATE COLUMNS' ) | TableName ( 'INDEX' IndexNameList? | AnalyzeColumnOption | 'PARTITION' PartitionNameList ( 'INDEX' IndexNameList? | AnalyzeColumnOption )? )? ) | 'INCREMENTAL' 'TABLE' TableName ( 'PARTITION' PartitionNameList )? 'INDEX' IndexNameList? ) AnalyzeOptionListOpt + 'ANALYZE' ( 'TABLE' ( TableNameList ( 'ALL COLUMNS' | 'PREDICATE COLUMNS' ) | TableName ( 'INDEX' IndexNameList? | AnalyzeColumnOption | 'PARTITION' PartitionNameList ( 'INDEX' IndexNameList? | AnalyzeColumnOption )? )? ) ) AnalyzeOptionListOpt AnalyzeOptionListOpt ::= ( WITH AnalyzeOptionList )? diff --git a/sql-statements/sql-statement-backup.md b/sql-statements/sql-statement-backup.md index 030f4689c16b5..b22e85dcc3ac5 100644 --- a/sql-statements/sql-statement-backup.md +++ b/sql-statements/sql-statement-backup.md @@ -1,16 +1,16 @@ --- title: BACKUP | TiDB SQL Statement Reference summary: An overview of the usage of BACKUP for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-backup/'] --- # BACKUP This statement is used to perform a distributed backup of the TiDB cluster. -> **Note:** +> **Warning:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> - This feature is experimental. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. +> - This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. The `BACKUP` statement uses the same engine as the [BR tool](https://docs.pingcap.com/tidb/stable/backup-and-restore-overview) does, except that the backup process is driven by TiDB itself rather than a separate BR tool. All benefits and warnings of BR also apply to this statement. diff --git a/sql-statements/sql-statement-batch.md b/sql-statements/sql-statement-batch.md index 3f8451a6a4865..ed87e0aa4f0f6 100644 --- a/sql-statements/sql-statement-batch.md +++ b/sql-statements/sql-statement-batch.md @@ -27,6 +27,10 @@ BATCH ON id LIMIT 1 INSERT INTO t SELECT t2.id, t2.v, t3.v FROM t2 JOIN t3 ON t2 Non-transactional DML, shard column must be fully specified ``` +> **Note:** +> +> The `BATCH` statement is internally rewritten and split into multiple DML statements during execution. In the current version, table aliases might not be preserved, which can result in errors such as `Unknown column '.' in 'where clause'`. To avoid this issue, do not use table aliases. Before execution, use `DRY RUN QUERY` or `DRY RUN` to preview the split statements before execution. For more information, see [Non-Transactional DML Statements](/non-transactional-dml.md). + ## Synopsis ```ebnf+diagram diff --git a/sql-statements/sql-statement-begin.md b/sql-statements/sql-statement-begin.md index 15ef1c9d3e771..ad78ec7921e0c 100644 --- a/sql-statements/sql-statement-begin.md +++ b/sql-statements/sql-statement-begin.md @@ -1,7 +1,6 @@ --- title: BEGIN | TiDB SQL Statement Reference summary: An overview of the usage of BEGIN for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-begin/','/docs/dev/reference/sql/statements/begin/'] --- # BEGIN diff --git a/sql-statements/sql-statement-calibrate-resource.md b/sql-statements/sql-statement-calibrate-resource.md index db8c8af8e7a1b..2081409d55aa4 100644 --- a/sql-statements/sql-statement-calibrate-resource.md +++ b/sql-statements/sql-statement-calibrate-resource.md @@ -9,7 +9,7 @@ The `CALIBRATE RESOURCE` statement is used to estimate and output the ['Request > **Note:** > -> This feature is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This feature is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). ## Synopsis @@ -28,7 +28,7 @@ To execute this command, make sure that the following requirements are met: - You have enabled [`tidb_enable_resource_control`](/system-variables.md#tidb_enable_resource_control-new-in-v660). - The user has `SUPER` or `RESOURCE_GROUP_ADMIN` privilege. -- The user has the `SELECT` privilege for all tables in the `METRICS_SCHEMA` schema. +- To [estimate capacity based on actual workload](#estimate-capacity-based-on-actual-workload), the user must have the `SELECT` privilege for all tables in the `METRICS_SCHEMA` schema. ## Methods for estimating capacity diff --git a/sql-statements/sql-statement-cancel-import-job.md b/sql-statements/sql-statement-cancel-import-job.md index fe0c632b72dda..d4b911d846384 100644 --- a/sql-statements/sql-statement-cancel-import-job.md +++ b/sql-statements/sql-statement-cancel-import-job.md @@ -7,12 +7,6 @@ summary: An overview of the usage of CANCEL IMPORT in TiDB. The `CANCEL IMPORT` statement is used to cancel a data import job created in TiDB. - - ## Required privileges To cancel a data import job, you need to be the creator of the import job or have the `SUPER` privilege. diff --git a/sql-statements/sql-statement-change-column.md b/sql-statements/sql-statement-change-column.md index 41d857ad79bfe..5edc05af5129d 100644 --- a/sql-statements/sql-statement-change-column.md +++ b/sql-statements/sql-statement-change-column.md @@ -1,7 +1,6 @@ --- title: CHANGE COLUMN | TiDB SQL Statement Reference summary: An overview of the usage of CHANGE COLUMN for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-change-column/','/docs/dev/reference/sql/statements/change-column/'] --- # CHANGE COLUMN @@ -152,7 +151,7 @@ ERROR 8200 (HY000): Unsupported modify column: change from original type decimal * Changes of [Reorg-Data](/sql-statements/sql-statement-modify-column.md#reorg-data-change) types on primary key columns are not supported. * Changes of column types on partitioned tables are not supported. * Changes of column types on generated columns are not supported. -* Changes of some data types (for example, some TIME, Bit, Set, Enum, and JSON types) are not supported due to the compatibility issues of the `CAST` function's behavior between TiDB and MySQL. +* Changes from some data types (for example, TIME, BIT, SET, ENUM, and JSON types) to some other types are not supported due to the compatibility issues of the `CAST` function's behavior between TiDB and MySQL. ## See also diff --git a/sql-statements/sql-statement-change-drainer.md b/sql-statements/sql-statement-change-drainer.md index 23562f711c42e..7a2887f6e2e3c 100644 --- a/sql-statements/sql-statement-change-drainer.md +++ b/sql-statements/sql-statement-change-drainer.md @@ -1,7 +1,6 @@ --- title: CHANGE DRAINER summary: An overview of the usage of CHANGE DRAINER for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-change-drainer/'] --- # CHANGE DRAINER @@ -10,7 +9,7 @@ The `CHANGE DRAINER` statement modifies the status information for Drainer in th > **Note:** > -> This feature is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This feature is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). > **Tip:** > diff --git a/sql-statements/sql-statement-change-pump.md b/sql-statements/sql-statement-change-pump.md index 2b2fe66db3260..e76276c35a104 100644 --- a/sql-statements/sql-statement-change-pump.md +++ b/sql-statements/sql-statement-change-pump.md @@ -1,7 +1,6 @@ --- title: CHANGE PUMP summary: An overview of the usage of CHANGE PUMP for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-change-pump/'] --- # CHANGE PUMP @@ -10,7 +9,7 @@ The `CHANGE PUMP` statement modifies the status information for Pump in the clus > **Note:** > -> This feature is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This feature is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). > **Tip:** > diff --git a/sql-statements/sql-statement-commit.md b/sql-statements/sql-statement-commit.md index 8543ab83a54bf..ff2f1a484fe27 100644 --- a/sql-statements/sql-statement-commit.md +++ b/sql-statements/sql-statement-commit.md @@ -1,7 +1,6 @@ --- title: COMMIT | TiDB SQL Statement Reference summary: An overview of the usage of COMMIT for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-commit/','/docs/dev/reference/sql/statements/commit/'] --- # COMMIT @@ -41,7 +40,7 @@ Query OK, 0 rows affected (0.01 sec) * Currently, TiDB use Metadata Locking (MDL) to prevent DDL statements from modifying tables used by transactions by default. The behavior of metadata lock is different between TiDB and MySQL. For more details, see [Metadata Lock](/metadata-lock.md). * By default, TiDB 3.0.8 and later versions use [Pessimistic Locking](/pessimistic-transaction.md). When using [Optimistic Locking](/optimistic-transaction.md), it is important to consider that a `COMMIT` statement might fail because rows have been modified by another transaction. -* When Optimistic Locking is enabled, `UNIQUE` and `PRIMARY KEY` constraint checks are deferred until statement commit. This results in additional situations where a a `COMMIT` statement might fail. This behavior can be changed by setting `tidb_constraint_check_in_place=ON`. +* When Optimistic Locking is enabled, `UNIQUE` and `PRIMARY KEY` constraint checks are deferred until statement commit. This results in additional situations where a `COMMIT` statement might fail. This behavior can be changed by setting `tidb_constraint_check_in_place=ON`. * TiDB parses but ignores the syntax `ROLLBACK AND [NO] RELEASE`. This functionality is used in MySQL to disconnect the client session immediately after committing the transaction. In TiDB, it is recommended to instead use the `mysql_close()` functionality of your client driver. * TiDB parses but ignores the syntax `ROLLBACK AND [NO] CHAIN`. This functionality is used in MySQL to immediately start a new transaction with the same isolation level while the current transaction is being committed. In TiDB, it is recommended to instead start a new transaction. diff --git a/sql-statements/sql-statement-create-binding.md b/sql-statements/sql-statement-create-binding.md index fcf3dc8ce70d8..ace22fde975ce 100644 --- a/sql-statements/sql-statement-create-binding.md +++ b/sql-statements/sql-statement-create-binding.md @@ -1,7 +1,6 @@ --- title: CREATE [GLOBAL|SESSION] BINDING summary: Use of CREATE BINDING in TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-create-binding/'] --- # CREATE [GLOBAL|SESSION] BINDING diff --git a/sql-statements/sql-statement-create-database.md b/sql-statements/sql-statement-create-database.md index 72c489bd8c8e2..1e27434343bca 100644 --- a/sql-statements/sql-statement-create-database.md +++ b/sql-statements/sql-statement-create-database.md @@ -1,7 +1,6 @@ --- title: CREATE DATABASE | TiDB SQL Statement Reference summary: An overview of the usage of CREATE DATABASE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-create-database/','/docs/dev/reference/sql/statements/create-database/'] --- # CREATE DATABASE diff --git a/sql-statements/sql-statement-create-index.md b/sql-statements/sql-statement-create-index.md index 501d4ffe19a0c..acf467fac48cb 100644 --- a/sql-statements/sql-statement-create-index.md +++ b/sql-statements/sql-statement-create-index.md @@ -1,13 +1,20 @@ --- title: CREATE INDEX | TiDB SQL Statement Reference summary: An overview of the usage of CREATE INDEX for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-create-index/','/docs/dev/reference/sql/statements/create-index/'] --- # CREATE INDEX This statement adds a new index to an existing table. It is an alternative syntax to `ALTER TABLE .. ADD INDEX`, and included for MySQL compatibility. + + +> **Note:** +> +> For [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters with 4 vCPU, it is recommended to manually disable [`tidb_ddl_enable_fast_reorg`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) to prevent resource limitations from affecting cluster stability during index creation. Disabling this setting allows indexes to be created using transactions, which reduces the overall impact on the cluster. + + + ## Synopsis ```ebnf+diagram diff --git a/sql-statements/sql-statement-create-placement-policy.md b/sql-statements/sql-statement-create-placement-policy.md index 7d0c7c0d77235..8f6f774358dae 100644 --- a/sql-statements/sql-statement-create-placement-policy.md +++ b/sql-statements/sql-statement-create-placement-policy.md @@ -9,7 +9,7 @@ summary: The usage of CREATE PLACEMENT POLICY in TiDB. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-create-resource-group.md b/sql-statements/sql-statement-create-resource-group.md index 6d34e7c1b464e..c31ce22d6d0c6 100644 --- a/sql-statements/sql-statement-create-resource-group.md +++ b/sql-statements/sql-statement-create-resource-group.md @@ -9,7 +9,7 @@ You can use the `CREATE RESOURCE GROUP` statement to create a resource group. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-create-sequence.md b/sql-statements/sql-statement-create-sequence.md index 01946a387eaba..9d482399a82b5 100644 --- a/sql-statements/sql-statement-create-sequence.md +++ b/sql-statements/sql-statement-create-sequence.md @@ -1,7 +1,6 @@ --- title: CREATE SEQUENCE summary: An overview of the usage of CREATE SEQUENCE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-create-sequence/','/docs/dev/reference/sql/statements/create-sequence/'] --- # CREATE SEQUENCE @@ -12,7 +11,7 @@ The `CREATE SEQUENCE` statement creates sequence objects in TiDB. The sequence i ```ebnf+diagram CreateSequenceStmt ::= - 'CREATE' 'SEQUENCE' IfNotExists TableName CreateSequenceOptionListOpt CreateTableOptionListOpt + 'CREATE' 'SEQUENCE' IfNotExists TableName CreateSequenceOptionListOpt IfNotExists ::= ('IF' 'NOT' 'EXISTS')? @@ -28,6 +27,7 @@ SequenceOptionList ::= SequenceOption ::= ( 'INCREMENT' ( '='? | 'BY' ) | 'START' ( '='? | 'WITH' ) | ( 'MINVALUE' | 'MAXVALUE' | 'CACHE' ) '='? ) SignedNum +| 'COMMENT' '='? stringLit | 'NOMINVALUE' | 'NO' ( 'MINVALUE' | 'MAXVALUE' | 'CACHE' | 'CYCLE' ) | 'NOMAXVALUE' @@ -69,7 +69,7 @@ You can control a sequence through the following expression functions: + `NEXTVAL` or `NEXT VALUE FOR` - Essentially, both are the `nextval()` function that gets the next valid value of a sequence object. The parameter of the `nextval()` function is the `identifier` of the sequence. + Essentially, both are the `NEXTVAL()` function that gets the next valid value of a sequence object. The parameter of the `NEXTVAL()` function is the `identifier` of the sequence. + `LASTVAL` @@ -97,51 +97,51 @@ You can control a sequence through the following expression functions: Query OK, 0 rows affected (0.06 sec) ``` -+ Use the `nextval()` function to get the next value of the sequence object: ++ Use the `NEXTVAL()` function to get the next value of the sequence object: {{< copyable "sql" >}} ```sql - SELECT nextval(seq); + SELECT NEXTVAL(seq); ``` ``` +--------------+ - | nextval(seq) | + | NEXTVAL(seq) | +--------------+ | 1 | +--------------+ 1 row in set (0.02 sec) ``` -+ Use the `lastval()` function to get the value generated by the last call to a sequence object in this session: ++ Use the `LASTVAL()` function to get the value generated by the last call to a sequence object in this session: {{< copyable "sql" >}} ```sql - SELECT lastval(seq); + SELECT LASTVAL(seq); ``` ``` +--------------+ - | lastval(seq) | + | LASTVAL(seq) | +--------------+ | 1 | +--------------+ 1 row in set (0.02 sec) ``` -+ Use the `setval()` function to set the current value (or the current position) of the sequence object: ++ Use the `SETVAL()` function to set the current value (or the current position) of the sequence object: {{< copyable "sql" >}} ```sql - SELECT setval(seq, 10); + SELECT SETVAL(seq, 10); ``` ``` +-----------------+ - | setval(seq, 10) | + | SETVAL(seq, 10) | +-----------------+ | 10 | +-----------------+ @@ -177,58 +177,58 @@ You can control a sequence through the following expression functions: Query OK, 0 rows affected (0.01 sec) ``` -+ When the sequence object has not been used in this session, the `lastval()` function returns a `NULL` value. ++ When the sequence object has not been used in this session, the `LASTVAL()` function returns a `NULL` value. {{< copyable "sql" >}} ```sql - SELECT lastval(seq2); + SELECT LASTVAL(seq2); ``` ``` +---------------+ - | lastval(seq2) | + | LASTVAL(seq2) | +---------------+ | NULL | +---------------+ 1 row in set (0.01 sec) ``` -+ The first valid value of the `nextval()` function for the sequence object is the value of `START` parameter. ++ The first valid value of the `NEXTVAL()` function for the sequence object is the value of `START` parameter. {{< copyable "sql" >}} ```sql - SELECT nextval(seq2); + SELECT NEXTVAL(seq2); ``` ``` +---------------+ - | nextval(seq2) | + | NEXTVAL(seq2) | +---------------+ | 3 | +---------------+ 1 row in set (0.00 sec) ``` -+ Although the `setval()` function can change the current value of the sequence object, it cannot change the arithmetic progression rule for the next value. ++ Although the `SETVAL()` function can change the current value of the sequence object, it cannot change the arithmetic progression rule for the next value. {{< copyable "sql" >}} ```sql - SELECT setval(seq2, 6); + SELECT SETVAL(seq2, 6); ``` ``` +-----------------+ - | setval(seq2, 6) | + | SETVAL(seq2, 6) | +-----------------+ | 6 | +-----------------+ 1 row in set (0.00 sec) ``` -+ When you use `nextval()` to get the next value, the next value will follow the arithmetic progression rule defined by the sequence. ++ When you use `NEXTVAL()` to get the next value, the next value will follow the arithmetic progression rule defined by the sequence. {{< copyable "sql" >}} @@ -306,7 +306,7 @@ For example: ``` 1, 3, 5, ... // The sequence starts from 1 and increments by 2. -select setval(seq, 6) // Sets the current value of a sequence to 6. +select SETVAL(seq, 6) // Sets the current value of a sequence to 6. 7, 9, 11, ... // Subsequent values still follow the progression rule. ``` @@ -314,5 +314,7 @@ In the `CYCLE` mode, the initial value of a sequence in the first round is the v ## See also +* [ALTER SEQUENCE](/sql-statements/sql-statement-alter-sequence.md) * [DROP SEQUENCE](/sql-statements/sql-statement-drop-sequence.md) * [SHOW CREATE SEQUENCE](/sql-statements/sql-statement-show-create-sequence.md) +* [Sequence Functions](/functions-and-operators/sequence-functions.md) diff --git a/sql-statements/sql-statement-create-table-like.md b/sql-statements/sql-statement-create-table-like.md index 912af59fbd0cb..331abd305abbb 100644 --- a/sql-statements/sql-statement-create-table-like.md +++ b/sql-statements/sql-statement-create-table-like.md @@ -1,7 +1,6 @@ --- title: CREATE TABLE LIKE | TiDB SQL Statement Reference summary: An overview of the usage of CREATE TABLE LIKE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-create-table-like/','/docs/dev/reference/sql/statements/create-table-like/'] --- # CREATE TABLE LIKE diff --git a/sql-statements/sql-statement-create-table.md b/sql-statements/sql-statement-create-table.md index b9651433d6b09..f09faf30d171e 100644 --- a/sql-statements/sql-statement-create-table.md +++ b/sql-statements/sql-statement-create-table.md @@ -1,7 +1,6 @@ --- title: CREATE TABLE | TiDB SQL Statement Reference summary: An overview of the usage of CREATE TABLE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-create-table/','/docs/dev/reference/sql/statements/create-table/'] --- # CREATE TABLE @@ -133,7 +132,7 @@ The following *table_options* are supported. Other options such as `AVG_ROW_LENG | `AUTO_INCREMENT` | The initial value of the increment field | `AUTO_INCREMENT` = 5 | | [`SHARD_ROW_ID_BITS`](/shard-row-id-bits.md)| To set the number of bits for the implicit `_tidb_rowid` shards |`SHARD_ROW_ID_BITS` = 4| |`PRE_SPLIT_REGIONS`| To pre-split `2^(PRE_SPLIT_REGIONS)` Regions when creating a table |`PRE_SPLIT_REGIONS` = 4| -|`AUTO_ID_CACHE`| To set the auto ID cache size in a TiDB instance. By default, TiDB automatically changes this size according to allocation speed of auto ID |`AUTO_ID_CACHE` = 200. Note that this option is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters.| +|`AUTO_ID_CACHE`| To set the auto ID cache size in a TiDB instance. By default, TiDB automatically changes this size according to allocation speed of auto ID |`AUTO_ID_CACHE` = 200 | |`AUTO_RANDOM_BASE`| To set the initial incremental part value of auto_random. This option can be considered as a part of the internal interface. Users can ignore this parameter |`AUTO_RANDOM_BASE` = 0| | `CHARACTER SET` | To specify the [character set](/character-set-and-collation.md) for the table | `CHARACTER SET` = 'utf8mb4' | | `COMMENT` | The comment information | `COMMENT` = 'comment info' | @@ -256,7 +255,7 @@ mysql> DESC t1; * The `[ASC | DESC]` in `index_col_name` is currently parsed but ignored (MySQL 5.7 compatible behavior). * The `COMMENT` attribute does not support the `WITH PARSER` option. * TiDB supports 1017 columns in a single table by default and 4096 columns at most. The corresponding number limit in InnoDB is 1017 columns, and the hard limit in MySQL is 4096 columns. For details, see [TiDB Limitations](/tidb-limitations.md). -* For partitioned tables, only Range, Hash and Range Columns (single column) are supported. For details, see [partitioned table](/partitioned-table.md). +* TiDB supports `HASH`, `RANGE`, `LIST`, and `KEY` [partitioning types](/partitioned-table.md#partitioning-types). For an unsupported partition type, TiDB returns `Warning: Unsupported partition type %s, treat as normal table`, where `%s` is the specific unsupported partition type. ## See also diff --git a/sql-statements/sql-statement-create-user.md b/sql-statements/sql-statement-create-user.md index 76d0c78569c07..241740a8d5a5d 100644 --- a/sql-statements/sql-statement-create-user.md +++ b/sql-statements/sql-statement-create-user.md @@ -1,7 +1,6 @@ --- title: CREATE USER | TiDB SQL Statement Reference summary: An overview of the usage of CREATE USER for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-create-user/','/docs/dev/reference/sql/statements/create-user/'] --- # CREATE USER @@ -20,6 +19,12 @@ IfNotExists ::= UserSpecList ::= UserSpec ( ',' UserSpec )* +RequireClauseOpt ::= + ( 'REQUIRE' 'NONE' | 'REQUIRE' 'SSL' | 'REQUIRE' 'X509' | 'REQUIRE' RequireList )? + +RequireList ::= + ( "ISSUER" stringLit | "SUBJECT" stringLit | "CIPHER" stringLit | "SAN" stringLit | "TOKEN_ISSUER" stringLit )* + UserSpec ::= Username AuthOption @@ -37,6 +42,10 @@ LockOption ::= ( 'ACCOUNT' 'LOCK' | 'ACCOUNT' 'UNLOCK' )? AttributeOption ::= ( 'COMMENT' CommentString | 'ATTRIBUTE' AttributeString )? ResourceGroupNameOption::= ( 'RESOURCE' 'GROUP' Identifier)? + +RequireClauseOpt ::= ('REQUIRE' ('NONE' | 'SSL' | 'X509' | RequireListElement ('AND'? RequireListElement)*))? + +RequireListElement ::= 'ISSUER' Issuer | 'SUBJECT' Subject | 'CIPHER' Cipher | 'SAN' SAN | 'TOKEN_ISSUER' TokenIssuer ``` ## Examples diff --git a/sql-statements/sql-statement-create-view.md b/sql-statements/sql-statement-create-view.md index 6a4b907678d42..64a3678af8ca1 100644 --- a/sql-statements/sql-statement-create-view.md +++ b/sql-statements/sql-statement-create-view.md @@ -1,7 +1,6 @@ --- title: CREATE VIEW | TiDB SQL Statement Reference summary: An overview of the usage of CREATE VIEW for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-create-view/','/docs/dev/reference/sql/statements/create-view/'] --- # CREATE VIEW diff --git a/sql-statements/sql-statement-deallocate.md b/sql-statements/sql-statement-deallocate.md index d88f9edb374fd..0c0728d1a42cf 100644 --- a/sql-statements/sql-statement-deallocate.md +++ b/sql-statements/sql-statement-deallocate.md @@ -1,7 +1,6 @@ --- title: DEALLOCATE | TiDB SQL Statement Reference summary: An overview of the usage of DEALLOCATE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-deallocate/','/docs/dev/reference/sql/statements/deallocate/'] --- # DEALLOCATE diff --git a/sql-statements/sql-statement-delete.md b/sql-statements/sql-statement-delete.md index 7bb1e491eaae2..140ddf397cfd8 100644 --- a/sql-statements/sql-statement-delete.md +++ b/sql-statements/sql-statement-delete.md @@ -1,7 +1,6 @@ --- title: DELETE | TiDB SQL Statement Reference summary: An overview of the usage of DELETE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-delete/','/docs/dev/reference/sql/statements/delete/'] --- # DELETE diff --git a/sql-statements/sql-statement-desc.md b/sql-statements/sql-statement-desc.md index 541f6eee2e0f3..bdc55b29da8ad 100644 --- a/sql-statements/sql-statement-desc.md +++ b/sql-statements/sql-statement-desc.md @@ -1,7 +1,6 @@ --- title: DESC | TiDB SQL Statement Reference summary: An overview of the usage of `DESC` for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-desc/','/docs/dev/reference/sql/statements/desc/'] --- # DESC diff --git a/sql-statements/sql-statement-describe.md b/sql-statements/sql-statement-describe.md index 009b9d6ea8db9..785b7d2372bb1 100644 --- a/sql-statements/sql-statement-describe.md +++ b/sql-statements/sql-statement-describe.md @@ -1,7 +1,6 @@ --- title: DESCRIBE | TiDB SQL Statement Reference summary: An overview of the usage of DESCRIBE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-describe/','/docs/dev/reference/sql/statements/describe/'] --- # DESCRIBE diff --git a/sql-statements/sql-statement-do.md b/sql-statements/sql-statement-do.md index 3bf40115429d4..a40447e3569bf 100644 --- a/sql-statements/sql-statement-do.md +++ b/sql-statements/sql-statement-do.md @@ -1,7 +1,6 @@ --- title: DO | TiDB SQL Statement Reference summary: An overview of the usage of DO for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-do/','/docs/dev/reference/sql/statements/do/'] --- # DO diff --git a/sql-statements/sql-statement-drop-binding.md b/sql-statements/sql-statement-drop-binding.md index b240e6293ea3a..dd5d997a40e45 100644 --- a/sql-statements/sql-statement-drop-binding.md +++ b/sql-statements/sql-statement-drop-binding.md @@ -1,7 +1,6 @@ --- title: DROP [GLOBAL|SESSION] BINDING summary: Use of DROP BINDING in TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-drop-binding/'] --- # DROP [GLOBAL|SESSION] BINDING @@ -166,7 +165,7 @@ mysql> SELECT @@LAST_PLAN_FROM_BINDING; +--------------------------+ 1 row in set (0.01 sec) -mysql> SHOW BINDINGS\G; +mysql> SHOW BINDINGS\G *************************** 1. row *************************** Original_sql: select * from `test` . `t` where `a` = ? Bind_sql: SELECT /*+ use_index(@`sel_1` `test`.`t` ) ignore_index(`t` `a`)*/ * FROM `test`.`t` WHERE `a` = 1 @@ -187,7 +186,7 @@ No query specified mysql> DROP BINDING FOR SQL DIGEST '6909a1bbce5f64ade0a532d7058dd77b6ad5d5068aee22a531304280de48349f'; Query OK, 0 rows affected (0.00 sec) -mysql> SHOW BINDINGS\G; +mysql> SHOW BINDINGS\G Empty set (0.01 sec) ERROR: diff --git a/sql-statements/sql-statement-drop-column.md b/sql-statements/sql-statement-drop-column.md index bf89165c62382..3c87062e5d354 100644 --- a/sql-statements/sql-statement-drop-column.md +++ b/sql-statements/sql-statement-drop-column.md @@ -1,7 +1,6 @@ --- title: DROP COLUMN | TiDB SQL Statement Reference summary: An overview of the usage of DROP COLUMN for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-drop-column/','/docs/dev/reference/sql/statements/drop-column/'] --- # DROP COLUMN diff --git a/sql-statements/sql-statement-drop-database.md b/sql-statements/sql-statement-drop-database.md index acb205a34a830..4a3ff9598a5e4 100644 --- a/sql-statements/sql-statement-drop-database.md +++ b/sql-statements/sql-statement-drop-database.md @@ -1,7 +1,6 @@ --- title: DROP DATABASE | TiDB SQL Statement Reference summary: An overview of the usage of DROP DATABASE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-drop-database/','/docs/dev/reference/sql/statements/drop-database/'] --- # DROP DATABASE diff --git a/sql-statements/sql-statement-drop-index.md b/sql-statements/sql-statement-drop-index.md index 613c35c528c59..982401b42109d 100644 --- a/sql-statements/sql-statement-drop-index.md +++ b/sql-statements/sql-statement-drop-index.md @@ -1,7 +1,6 @@ --- title: DROP INDEX | TiDB SQL Statement Reference summary: An overview of the usage of DROP INDEX for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-drop-index/','/docs/dev/reference/sql/statements/drop-index/'] --- # DROP INDEX @@ -63,7 +62,7 @@ Query OK, 0 rows affected (0.30 sec) ## See also -* [SHOW INDEX](/sql-statements/sql-statement-show-index.md) +* [SHOW INDEXES](/sql-statements/sql-statement-show-indexes.md) * [CREATE INDEX](/sql-statements/sql-statement-create-index.md) * [ADD INDEX](/sql-statements/sql-statement-add-index.md) * [RENAME INDEX](/sql-statements/sql-statement-rename-index.md) diff --git a/sql-statements/sql-statement-drop-placement-policy.md b/sql-statements/sql-statement-drop-placement-policy.md index 2e3f9e610dd8f..851562ede2872 100644 --- a/sql-statements/sql-statement-drop-placement-policy.md +++ b/sql-statements/sql-statement-drop-placement-policy.md @@ -9,7 +9,7 @@ summary: The usage of ALTER PLACEMENT POLICY in TiDB. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-drop-resource-group.md b/sql-statements/sql-statement-drop-resource-group.md index e42cc8e525ae6..b53013b39989e 100644 --- a/sql-statements/sql-statement-drop-resource-group.md +++ b/sql-statements/sql-statement-drop-resource-group.md @@ -9,7 +9,7 @@ You can use the `DROP RESOURCE GROUP` statement to drop a resource group. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-drop-sequence.md b/sql-statements/sql-statement-drop-sequence.md index ca346c168859f..b5f944a6f0bce 100644 --- a/sql-statements/sql-statement-drop-sequence.md +++ b/sql-statements/sql-statement-drop-sequence.md @@ -1,7 +1,6 @@ --- title: DROP SEQUENCE summary: An overview of the usage of DROP SEQUENCE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-drop-sequence/','/docs/dev/reference/sql/statements/drop-sequence/'] --- # DROP SEQUENCE @@ -52,4 +51,5 @@ This statement is a TiDB extension. The implementation is modeled on sequences a ## See also * [CREATE SEQUENCE](/sql-statements/sql-statement-create-sequence.md) +* [ALTER SEQUENCE](/sql-statements/sql-statement-alter-sequence.md) * [SHOW CREATE SEQUENCE](/sql-statements/sql-statement-show-create-sequence.md) diff --git a/sql-statements/sql-statement-drop-stats.md b/sql-statements/sql-statement-drop-stats.md index 86ddcfd01bf93..65d1326e5e101 100644 --- a/sql-statements/sql-statement-drop-stats.md +++ b/sql-statements/sql-statement-drop-stats.md @@ -1,7 +1,6 @@ --- title: DROP STATS summary: An overview of the usage of DROP STATS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-drop-stats/'] --- # DROP STATS diff --git a/sql-statements/sql-statement-drop-table.md b/sql-statements/sql-statement-drop-table.md index 8024cba7a83da..585ba050f9341 100644 --- a/sql-statements/sql-statement-drop-table.md +++ b/sql-statements/sql-statement-drop-table.md @@ -1,7 +1,6 @@ --- title: DROP TABLE | TiDB SQL Statement Reference summary: An overview of the usage of DROP TABLE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-drop-table/','/docs/dev/reference/sql/statements/drop-table/'] --- # DROP TABLE diff --git a/sql-statements/sql-statement-drop-user.md b/sql-statements/sql-statement-drop-user.md index e87ae43ac2324..c735c5e275d49 100644 --- a/sql-statements/sql-statement-drop-user.md +++ b/sql-statements/sql-statement-drop-user.md @@ -1,7 +1,6 @@ --- title: DROP USER | TiDB SQL Statement Reference summary: An overview of the usage of DROP USER for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-drop-user/','/docs/dev/reference/sql/statements/drop-user/'] --- # DROP USER diff --git a/sql-statements/sql-statement-drop-view.md b/sql-statements/sql-statement-drop-view.md index 6201a03ddf301..c657f5b8cba6d 100644 --- a/sql-statements/sql-statement-drop-view.md +++ b/sql-statements/sql-statement-drop-view.md @@ -1,7 +1,6 @@ --- title: DROP VIEW | TiDB SQL Statement Reference summary: An overview of the usage of DROP VIEW for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-drop-view/','/docs/dev/reference/sql/statements/drop-view/'] --- # DROP VIEW diff --git a/sql-statements/sql-statement-execute.md b/sql-statements/sql-statement-execute.md index d2366d88bd736..86dd05635665b 100644 --- a/sql-statements/sql-statement-execute.md +++ b/sql-statements/sql-statement-execute.md @@ -1,7 +1,6 @@ --- title: EXECUTE | TiDB SQL Statement Reference summary: An overview of the usage of EXECUTE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-execute/','/docs/dev/reference/sql/statements/execute/'] --- # EXECUTE diff --git a/sql-statements/sql-statement-explain-analyze.md b/sql-statements/sql-statement-explain-analyze.md index 462b0c8d84ec9..2042d5a8a00b0 100644 --- a/sql-statements/sql-statement-explain-analyze.md +++ b/sql-statements/sql-statement-explain-analyze.md @@ -1,7 +1,6 @@ --- title: EXPLAIN ANALYZE | TiDB SQL Statement Reference summary: An overview of the usage of EXPLAIN ANALYZE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-explain-analyze/','/docs/dev/reference/sql/statements/explain-analyze/'] --- # EXPLAIN ANALYZE @@ -34,7 +33,7 @@ ExplainableStmt ::= ## EXPLAIN ANALYZE output format -Different from `EXPLAIN`, `EXPLAIN ANALYZE` executes the corresponding SQL statement, records its runtime information, and returns the information together with the execution plan. Therefore, you can regard `EXPLAIN ANALYZE` as an extension of the `EXPLAIN` statement. Compared to `EXPLAIN` (for debugging query exeuction), the return results of `EXPLAIN ANALYZE` also include columns of information such as `actRows`, `execution info`, `memory`, and `disk`. The details of these columns are shown as follows: +Different from `EXPLAIN`, `EXPLAIN ANALYZE` executes the corresponding SQL statement, records its runtime information, and returns the information together with the execution plan. Therefore, you can regard `EXPLAIN ANALYZE` as an extension of the `EXPLAIN` statement. Compared to `EXPLAIN` (for debugging query execution), the return results of `EXPLAIN ANALYZE` also include columns of information such as `actRows`, `execution info`, `memory`, and `disk`. The details of these columns are shown as follows: | attribute name | description | |:----------------|:---------------------------------| @@ -91,7 +90,7 @@ EXPLAIN ANALYZE SELECT * FROM t1; +-------------------+----------+---------+-----------+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------+-----------+------+ | id | estRows | actRows | task | access object | execution info | operator info | memory | disk | +-------------------+----------+---------+-----------+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------+-----------+------+ -| TableReader_5 | 10000.00 | 3 | root | | time:278.2µs, loops:2, cop_task: {num: 1, max: 437.6µs, proc_keys: 3, rpc_num: 1, rpc_time: 423.9µs, copr_cache_hit_ratio: 0.00} | data:TableFullScan_4 | 251 Bytes | N/A | +| TableReader_5 | 10000.00 | 3 | root | | time:278.2µs, loops:2, cop_task: {num: 1, max: 437.6µs, proc_keys: 3, copr_cache_hit_ratio: 0.00}, rpc_info:{Cop:{num_rpc:1, total_time:423.9µs}} | data:TableFullScan_4 | 251 Bytes | N/A | | └─TableFullScan_4 | 10000.00 | 3 | cop[tikv] | table:t1 | tikv_task:{time:0s, loops:1}, scan_detail: {total_process_keys: 3, total_process_keys_size: 111, total_keys: 4, rocksdb: {delete_skipped_count: 0, key_skipped_count: 3, block: {cache_hit_count: 0, read_count: 0, read_byte: 0 Bytes}}} | keep order:false, stats:pseudo | N/A | N/A | +-------------------+----------+---------+-----------+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------+-----------+------+ 2 rows in set (0.00 sec) @@ -120,15 +119,15 @@ The execution information of the `Batch_Point_Get` operator is similar to that o The execution information of a `TableReader` operator is typically as follows: ``` -cop_task: {num: 6, max: 1.07587ms, min: 844.312µs, avg: 919.601µs, p95: 1.07587ms, max_proc_keys: 16, p95_proc_keys: 16, tot_proc: 1ms, tot_wait: 1ms, rpc_num: 6, rpc_time: 5.313996 ms, copr_cache_hit_ratio: 0.00} +cop_task: {num: 6, max: 1.07587ms, min: 844.312µs, avg: 919.601µs, p95: 1.07587ms, max_proc_keys: 16, p95_proc_keys: 16, tot_proc: 1ms, tot_wait: 1ms, copr_cache_hit_ratio: 0.00}, rpc_info:{Cop:{num_rpc:6, total_time:5.313996ms}} ``` - `cop_task`: Contains the execution information of `cop` tasks. For example: - `num`: The number of cop tasks. - `max`, `min`, `avg`, `p95`: The maximum, minimum, average, and P95 values of the execution time consumed for executing cop tasks. - `max_proc_keys` and `p95_proc_keys`: The maximum and P95 key-values scanned by TiKV in all cop tasks. If the difference between the maximum value and the P95 value is large, the data distribution might be imbalanced. - - `rpc_num`, `rpc_time`: The total number and total time consumed for `Cop` RPC requests sent to TiKV. - `copr_cache_hit_ratio`: The hit rate of Coprocessor Cache for `cop` task requests. +- `rpc_info`: The total number and total time of RPC requests sent to TiKV aggregated by request type. - `backoff`: Contains different types of backoff and the total waiting time of backoff. ### Insert diff --git a/sql-statements/sql-statement-explain.md b/sql-statements/sql-statement-explain.md index e74af4f93c03b..d1c26af49be05 100644 --- a/sql-statements/sql-statement-explain.md +++ b/sql-statements/sql-statement-explain.md @@ -1,12 +1,11 @@ --- title: EXPLAIN | TiDB SQL Statement Reference summary: An overview of the usage of EXPLAIN for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-explain/','/docs/dev/reference/sql/statements/explain/'] --- # `EXPLAIN` -The `EXPLAIN` statement shows the execution plan for a query without executing it. It is complimented by `EXPLAIN ANALYZE` which will execute the query. If the output of `EXPLAIN` does not match the expected result, consider executing `ANALYZE TABLE` on each table in the query. +The `EXPLAIN` statement shows the execution plan for a query without executing it. It complements the `EXPLAIN ANALYZE` statement, which executes the query. If the output of `EXPLAIN` does not match the expected result, consider executing `ANALYZE TABLE` on each table in the query to make sure the table statistics are up to date. The statements `DESC` and `DESCRIBE` are aliases of this statement. The alternative usage of `EXPLAIN ` is documented under [`SHOW [FULL] COLUMNS FROM`](/sql-statements/sql-statement-show-columns-from.md). @@ -192,10 +191,12 @@ To specify the format of the `EXPLAIN` output, you can use the `FORMAT = xxx` sy | FORMAT | Description | | ------ | ------ | | Not specified | If the format is not specified, `EXPLAIN` uses the default format `row`. | -| `row` | The `EXPLAIN` statement outputs results in a tabular format. See [Understand the Query Execution Plan](/explain-overview.md) for more information. | -| `brief` | The operator IDs in the output of the `EXPLAIN` statement are simplified, compared with those when `FORMAT` is left unspecified. | -| `dot` | The `EXPLAIN` statement outputs DOT execution plans, which can be used to generate PNG files through a `dot` program (in the `graphviz` package). | -| `tidb_json` | The `EXPLAIN` statement outputs execution plans in JSON and stores the operator information in a JSON array. | +| `brief` | The operator IDs in the output of the `EXPLAIN` statement are simplified, compared with those when `FORMAT` is left unspecified. | +| `dot` | The `EXPLAIN` statement outputs DOT execution plans, which can be used to generate PNG files through a `dot` program (in the `graphviz` package). | +| `row` | The `EXPLAIN` statement outputs results in a tabular format. See [Understand the Query Execution Plan](/explain-overview.md) for more information. | +| `tidb_json` | The `EXPLAIN` statement outputs execution plans in JSON and stores the operator information in a JSON array. | +| `verbose` | The `EXPLAIN` statement outputs results in the `row` format, with an additional `estCost` column for the estimated cost of the query in the results. For more information about how to use this format, see [SQL Plan Management](/sql-plan-management.md). | +| `plan_cache` | The `EXPLAIN` statement outputs results in the `row` format, with the [Plan Cache](/sql-non-prepared-plan-cache.md#diagnostics) information as a warning. | @@ -334,7 +335,7 @@ In the output, `id`, `estRows`, `taskType`, `accessObject`, and `operatorInfo` h ## MySQL compatibility -* Both the format of `EXPLAIN` and the potential execution plans in TiDB differ substaintially from MySQL. +* Both the format of `EXPLAIN` and the potential execution plans in TiDB differ substantially from MySQL. * TiDB does not support the `FORMAT=JSON` or `FORMAT=TREE` options. * `FORMAT=tidb_json` in TiDB is the JSON format output of the default `EXPLAIN` result. The format and fields are different from the `FORMAT=JSON` output in MySQL. diff --git a/sql-statements/sql-statement-flashback-to-timestamp.md b/sql-statements/sql-statement-flashback-cluster.md similarity index 58% rename from sql-statements/sql-statement-flashback-to-timestamp.md rename to sql-statements/sql-statement-flashback-cluster.md index 393b60aacfbea..3a7e5bc3e3414 100644 --- a/sql-statements/sql-statement-flashback-to-timestamp.md +++ b/sql-statements/sql-statement-flashback-cluster.md @@ -1,15 +1,23 @@ --- -title: FLASHBACK CLUSTER TO TIMESTAMP -summary: Learn the usage of FLASHBACK CLUSTER TO TIMESTAMP in TiDB databases. +title: FLASHBACK CLUSTER +summary: Learn the usage of FLASHBACK CLUSTER in TiDB databases. +aliases: ['/tidb/v7.5/sql-statement-flashback-to-timestamp','/tidb/stable/sql-statement-flashback-to-timestamp','/tidbcloud/sql-statement-flashback-to-timestamp'] --- -# FLASHBACK CLUSTER TO TIMESTAMP +# FLASHBACK CLUSTER -TiDB v6.4.0 introduces the `FLASHBACK CLUSTER TO TIMESTAMP` syntax. You can use it to restore a cluster to a specific point in time. +TiDB v6.4.0 introduces the `FLASHBACK CLUSTER TO TIMESTAMP` syntax. You can use it to restore a cluster to a specific point in time. When specifying the timestamp, you can either set a datetime value or use a time function. The format of datetime is like '2016-10-08 16:45:26.999', with millisecond as the minimum time unit. But in most cases, specifying the timestamp with second as the time unit is sufficient, for example, '2016-10-08 16:45:26'. + +Starting from v6.5.6, v7.1.3, and v7.5.1, TiDB introduces the `FLASHBACK CLUSTER TO TSO` syntax. This syntax enables you to use [TSO](/tso.md) to specify a more precise recovery point in time, thereby enhancing flexibility in data recovery. > **Warning:** > -> The `FLASHBACK CLUSTER TO TIMESTAMP` syntax is not applicable to [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. To avoid unexpected results, do not execute this statement on TiDB Serverless clusters. +> The `FLASHBACK CLUSTER TO [TIMESTAMP|TSO]` syntax is not applicable to [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. To avoid unexpected results, do not execute this statement on {{{ .starter }}} and {{{ .essential }}} clusters. + +> **Warning:** +> +> - When specifying a recovery point in time, make sure to check the validity of your target timestamp or TSO and avoid specifying a future time that exceeds the maximum TSO currently allocated by PD (see `Current TSO` on the Grafana PD panel). Otherwise, concurrent processing linear consistency and transaction isolation levels might be violated, leading to serious data correctness issues. +> - During `FLASHBACK CLUSTER` execution, the data cleanup process does not guarantee transaction consistency. After `FLASHBACK CLUSTER` completes, if you intend to use any historical version reading features in TiDB (such as [Stale Read](/stale-read.md) or [`tidb_snapshot`](/read-historical-data.md)), make sure that the specified historical timestamp falls outside the `FLASHBACK CLUSTER` execution period. Reading a historical version that includes data not fully restored by FLASHBACK might violate concurrent processing linear consistency and transaction isolation levels, leading to serious data correctness issues. @@ -23,19 +31,20 @@ TiDB v6.4.0 introduces the `FLASHBACK CLUSTER TO TIMESTAMP` syntax. You can use > **Note:** > -> The working principle of `FLASHBACK CLUSTER TO TIMESTAMP` is to write the old data of a specific point in time with the latest timestamp, and will not delete the current data. So before using this feature, you need to ensure that there is enough storage space for the old data and the current data. +> The working principle of `FLASHBACK CLUSTER TO [TIMESTAMP|TSO]` is to write the old data of a specific point in time with the latest timestamp, and will not delete the current data. So before using this feature, you need to ensure that there is enough storage space for the old data and the current data. ## Syntax ```sql FLASHBACK CLUSTER TO TIMESTAMP '2022-09-21 16:02:50'; +FLASHBACK CLUSTER TO TSO 445494839813079041; ``` ### Synopsis ```ebnf+diagram -FlashbackToTimestampStmt ::= - "FLASHBACK" "CLUSTER" "TO" "TIMESTAMP" stringLit +FlashbackToTimestampStmt + ::= 'FLASHBACK' 'CLUSTER' 'TO' ('TIMESTAMP' stringLit | 'TSO' LengthNum) ``` ## Notes @@ -51,8 +60,8 @@ FlashbackToTimestampStmt ::= * Only a user with the `SUPER` privilege can execute the `FLASHBACK CLUSTER` SQL statement. * `FLASHBACK CLUSTER` does not support rolling back DDL statements that modify PD-related information, such as `ALTER TABLE ATTRIBUTE`, `ALTER TABLE REPLICA`, and `CREATE PLACEMENT POLICY`. * At the time specified in the `FLASHBACK` statement, there cannot be a DDL statement that is not completely executed. If such a DDL exists, TiDB will reject it. -* Before executing `FLASHBACK CLUSTER TO TIMESTAMP`, TiDB disconnects all related connections and prohibits read and write operations on these tables until the `FLASHBACK CLUSTER` statement is completed. -* The `FLASHBACK CLUSTER TO TIMESTAMP` statement cannot be canceled after being executed. TiDB will keep retrying until it succeeds. +* Before executing `FLASHBACK CLUSTER`, TiDB disconnects all related connections and prohibits read and write operations on these tables until the `FLASHBACK CLUSTER` statement is completed. +* The `FLASHBACK CLUSTER` statement cannot be canceled after being executed. TiDB will keep retrying until it succeeds. * During the execution of `FLASHBACK CLUSTER`, if you need to back up data, you can only use [Backup & Restore](/br/br-snapshot-guide.md) and specify a `BackupTS` that is earlier than the start time of `FLASHBACK CLUSTER`. In addition, during the execution of `FLASHBACK CLUSTER`, enabling [log backup](/br/br-pitr-guide.md) will fail. Therefore, try to enable log backup after `FLASHBACK CLUSTER` is completed. * If the `FLASHBACK CLUSTER` statement causes the rollback of metadata (table structure, database structure), the related modifications will **not** be replicated by TiCDC. Therefore, you need to pause the task manually, wait for the completion of `FLASHBACK CLUSTER`, and manually replicate the schema definitions of the upstream and downstream to make sure that they are consistent. After that, you need to recreate the TiCDC changefeed. @@ -63,15 +72,15 @@ FlashbackToTimestampStmt ::= * Only a user with the `SUPER` privilege can execute the `FLASHBACK CLUSTER` SQL statement. * `FLASHBACK CLUSTER` does not support rolling back DDL statements that modify PD-related information, such as `ALTER TABLE ATTRIBUTE`, `ALTER TABLE REPLICA`, and `CREATE PLACEMENT POLICY`. * At the time specified in the `FLASHBACK` statement, there cannot be a DDL statement that is not completely executed. If such a DDL exists, TiDB will reject it. -* Before executing `FLASHBACK CLUSTER TO TIMESTAMP`, TiDB disconnects all related connections and prohibits read and write operations on these tables until the `FLASHBACK CLUSTER` statement is completed. -* The `FLASHBACK CLUSTER TO TIMESTAMP` statement cannot be canceled after being executed. TiDB will keep retrying until it succeeds. +* Before executing `FLASHBACK CLUSTER`, TiDB disconnects all related connections and prohibits read and write operations on these tables until the `FLASHBACK CLUSTER` statement is completed. +* The `FLASHBACK CLUSTER` statement cannot be canceled after being executed. TiDB will keep retrying until it succeeds. * If the `FLASHBACK CLUSTER` statement causes the rollback of metadata (table structure, database structure), the related modifications will **not** be replicated by TiCDC. Therefore, you need to pause the task manually, wait for the completion of `FLASHBACK CLUSTER`, and manually replicate the schema definitions of the upstream and downstream to make sure that they are consistent. After that, you need to recreate the TiCDC changefeed. ## Example -The following example shows how to restore the newly inserted data: +The following example shows how to flashback a cluster to a specific timestamp to restore newly inserted data: ```sql mysql> CREATE TABLE t(a INT); @@ -106,6 +115,52 @@ mysql> SELECT * FROM t; Empty set (0.00 sec) ``` +The following example shows how to flashback a cluster to a specific TSO to precisely restore mistakenly deleted data: + +```sql +mysql> INSERT INTO t VALUES (1); +Query OK, 1 row affected (0.02 sec) + +mysql> SELECT * FROM t; ++------+ +| a | ++------+ +| 1 | ++------+ +1 row in set (0.01 sec) + + +mysql> BEGIN; +Query OK, 0 rows affected (0.00 sec) + +mysql> SELECT @@tidb_current_ts; -- Get the current TSO ++--------------------+ +| @@tidb_current_ts | ++--------------------+ +| 446113975683252225 | ++--------------------+ +1 row in set (0.00 sec) + +mysql> ROLLBACK; +Query OK, 0 rows affected (0.00 sec) + + +mysql> DELETE FROM t; +Query OK, 1 rows affected (0.00 sec) + + +mysql> FLASHBACK CLUSTER TO TSO 446113975683252225; +Query OK, 0 rows affected (0.20 sec) + +mysql> SELECT * FROM t; ++------+ +| a | ++------+ +| 1 | ++------+ +1 row in set (0.01 sec) +``` + If there is a DDL statement that is not completely executed at the time specified in the `FLASHBACK` statement, the `FLASHBACK` statement fails: ```sql diff --git a/sql-statements/sql-statement-flashback-table.md b/sql-statements/sql-statement-flashback-table.md index 3cdfcb373720e..12bf536b9a15a 100644 --- a/sql-statements/sql-statement-flashback-table.md +++ b/sql-statements/sql-statement-flashback-table.md @@ -1,7 +1,6 @@ --- title: FLASHBACK TABLE summary: Learn how to recover tables using the `FLASHBACK TABLE` statement. -aliases: ['/docs/dev/sql-statements/sql-statement-flashback-table/','/docs/dev/reference/sql/statements/flashback-table/'] --- # FLASHBACK TABLE diff --git a/sql-statements/sql-statement-flush-privileges.md b/sql-statements/sql-statement-flush-privileges.md index 16f0a5dffdc47..13c3bc2dcee01 100644 --- a/sql-statements/sql-statement-flush-privileges.md +++ b/sql-statements/sql-statement-flush-privileges.md @@ -1,7 +1,6 @@ --- title: FLUSH PRIVILEGES | TiDB SQL Statement Reference summary: An overview of the usage of FLUSH PRIVILEGES for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-flush-privileges/','/docs/dev/reference/sql/statements/flush-privileges/'] --- # FLUSH PRIVILEGES diff --git a/sql-statements/sql-statement-flush-status.md b/sql-statements/sql-statement-flush-status.md index a55a69bcec5c1..619e4a703ef48 100644 --- a/sql-statements/sql-statement-flush-status.md +++ b/sql-statements/sql-statement-flush-status.md @@ -1,7 +1,6 @@ --- title: FLUSH STATUS | TiDB SQL Statement Reference summary: An overview of the usage of FLUSH STATUS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-flush-status/','/docs/dev/reference/sql/statements/flush-status/'] --- # FLUSH STATUS @@ -74,8 +73,9 @@ mysql> show status; ## MySQL compatibility -* This statement is included only for compatibility with MySQL. +* This statement is compatible with MySQL. ## See also * [SHOW \[GLOBAL|SESSION\] STATUS](/sql-statements/sql-statement-show-status.md) +* [Server Status Variables](/status-variables.md) diff --git a/sql-statements/sql-statement-flush-tables.md b/sql-statements/sql-statement-flush-tables.md index 4ce04037238fe..3d14ed1eb28fd 100644 --- a/sql-statements/sql-statement-flush-tables.md +++ b/sql-statements/sql-statement-flush-tables.md @@ -1,7 +1,6 @@ --- title: FLUSH TABLES | TiDB SQL Statement Reference summary: An overview of the usage of FLUSH TABLES for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-flush-tables/','/docs/dev/reference/sql/statements/flush-tables/'] --- # FLUSH TABLES diff --git a/sql-statements/sql-statement-grant-privileges.md b/sql-statements/sql-statement-grant-privileges.md index b8b36190047f6..83b14b0f76581 100644 --- a/sql-statements/sql-statement-grant-privileges.md +++ b/sql-statements/sql-statement-grant-privileges.md @@ -1,7 +1,6 @@ --- title: GRANT | TiDB SQL Statement Reference summary: An overview of the usage of GRANT for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-grant-privileges/','/docs/dev/reference/sql/statements/grant-privileges/'] --- # `GRANT ` @@ -55,6 +54,10 @@ PrivLevel ::= UserSpecList ::= UserSpec ( ',' UserSpec )* + +RequireClauseOpt ::= ('REQUIRE' ('NONE' | 'SSL' | 'X509' | RequireListElement ('AND'? RequireListElement)*))? + +RequireListElement ::= 'ISSUER' Issuer | 'SUBJECT' Subject | 'CIPHER' Cipher | 'SAN' SAN | 'TOKEN_ISSUER' TokenIssuer ``` ## Examples diff --git a/sql-statements/sql-statement-import-into.md b/sql-statements/sql-statement-import-into.md index 225a8a9fc3f10..b9f97fa36e5b5 100644 --- a/sql-statements/sql-statement-import-into.md +++ b/sql-statements/sql-statement-import-into.md @@ -5,40 +5,37 @@ summary: An overview of the usage of IMPORT INTO in TiDB. # IMPORT INTO -The `IMPORT INTO` statement is used to import data in formats such as `CSV`, `SQL`, and `PARQUET` into an empty table in TiDB via the [Physical Import Mode](/tidb-lightning/tidb-lightning-physical-import-mode.md) of TiDB Lightning. +The `IMPORT INTO` statement is used to import data in formats such as `CSV`, `SQL`, and `PARQUET` into an empty table in TiDB via the [Physical Import Mode](https://docs.pingcap.com/tidb/stable/tidb-lightning-physical-import-mode) of TiDB Lightning. - - -> **Warning:** -> -> Currently, this statement is experimental. It is not recommended to use it in production environments. - -`IMPORT INTO` supports importing data from files stored in Amazon S3, GCS, and the TiDB local storage. - -- For data files stored in Amazon S3, GCS, or Azure Blob Storage, `IMPORT INTO` supports running in the [TiDB backend task distributed execution framework](/tidb-distributed-execution-framework.md). - - - When this framework is enabled ([tidb_enable_dist_task](/system-variables.md#tidb_enable_dist_task-new-in-v710) is `ON`), `IMPORT INTO` splits a data import job into multiple sub-jobs and distributes these sub-jobs to different TiDB nodes for execution to improve the import efficiency. - - When this framework is disabled, `IMPORT INTO` only supports running on the TiDB node where the current user is connected. + - When this DXF is enabled ([tidb_enable_dist_task](/system-variables.md#tidb_enable_dist_task-new-in-v710) is `ON`), `IMPORT INTO` splits a data import job into multiple sub-jobs and distributes these sub-jobs to different TiDB nodes for execution to improve the import efficiency. + - When this DXF is disabled, `IMPORT INTO` only supports running on the TiDB node where the current user is connected. - For data files stored locally in TiDB, `IMPORT INTO` only supports running on the TiDB node where the current user is connected. Therefore, the data files need to be placed on the TiDB node where the current user is connected. If you access TiDB through a proxy or load balancer, you cannot import data files stored locally in TiDB. ## Restrictions -- Currently, `IMPORT INTO` supports importing data within 1 TiB. +- For TiDB Self-Managed, `IMPORT INTO` supports importing data within 10 TiB. For [TiDB Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-dedicated), `IMPORT INTO` supports importing data within 50 GiB. - `IMPORT INTO` only supports importing data into existing empty tables in the database. +- `IMPORT INTO` does not support importing data into an empty partition if other partitions of the same table already contain data. The target table must be completely empty for import operations. +- `IMPORT INTO` does not support importing data into a [temporary table](/temporary-tables.md) or a [cached table](/cached-tables.md). - `IMPORT INTO` does not support transactions or rollback. Executing `IMPORT INTO` within an explicit transaction (`BEGIN`/`END`) will return an error. - The execution of `IMPORT INTO` blocks the current connection until the import is completed. To execute the statement asynchronously, you can add the `DETACHED` option. -- `IMPORT INTO` does not support working simultaneously with features such as [Backup & Restore](/br/backup-and-restore-overview.md), [`FLASHBACK CLUSTER TO TIMESTAMP`](/sql-statements/sql-statement-flashback-to-timestamp.md), [acceleration of adding indexes](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630), data import using TiDB Lightning, data replication using TiCDC, or [Point-in-Time Recovery (PITR)](/br/br-log-architecture.md). +- `IMPORT INTO` does not support working simultaneously with features such as [Backup & Restore](https://docs.pingcap.com/tidb/stable/backup-and-restore-overview), [`FLASHBACK CLUSTER`](/sql-statements/sql-statement-flashback-cluster.md), [acceleration of adding indexes](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630), data import using TiDB Lightning, data replication using TiCDC, or [Point-in-Time Recovery (PITR)](https://docs.pingcap.com/tidb/stable/br-log-architecture). - Only one `IMPORT INTO` job can run on a cluster at a time. Although `IMPORT INTO` performs a precheck for running jobs, it is not a hard limit. Starting multiple import jobs might work when multiple clients execute `IMPORT INTO` simultaneously, but you need to avoid that because it might result in data inconsistency or import failures. - During the data import process, do not perform DDL or DML operations on the target table, and do not execute [`FLASHBACK DATABASE`](/sql-statements/sql-statement-flashback-database.md) for the target database. These operations can lead to import failures or data inconsistencies. In addition, it is **NOT** recommended to perform read operations during the import process, as the data being read might be inconsistent. Perform read and write operations only after the import is completed. -- The import process consumes system resources significantly. To get better performance, it is recommended to use TiDB nodes with at least 32 cores and 64 GiB of memory. TiDB writes sorted data to the TiDB [temporary directory](/tidb-configuration-file.md#temp-dir-new-in-v630) during import, so it is recommended to configure high-performance storage media such as flash memory. For more information, see [Physical Import Mode limitations](/tidb-lightning/tidb-lightning-physical-import-mode.md#requirements-and-restrictions). -- The TiDB [temporary directory](/tidb-configuration-file.md#temp-dir-new-in-v630) is expected to have at least 90 GiB of available space. It is recommended to allocate storage space that is equal to or greater than the volume of data to be imported. +- The import process consumes system resources significantly. For TiDB Self-Managed, to get better performance, it is recommended to use TiDB nodes with at least 32 cores and 64 GiB of memory. TiDB writes sorted data to the TiDB [temporary directory](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#temp-dir-new-in-v630) during import, so it is recommended to configure high-performance storage media for TiDB Self-Managed, such as flash memory. For more information, see [Physical Import Mode limitations](https://docs.pingcap.com/tidb/stable/tidb-lightning-physical-import-mode#requirements-and-restrictions). +- For TiDB Self-Managed, the TiDB [temporary directory](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#temp-dir-new-in-v630) is expected to have at least 90 GiB of available space. It is recommended to allocate storage space that is equal to or greater than the volume of data to be imported. - One import job supports importing data into one target table only. To import data into multiple target tables, after the import for a target table is completed, you need to create a new job for the next target table. - `IMPORT INTO` is not supported during TiDB cluster upgrades. +- When the [Global Sort](/tidb-global-sort.md) feature is used for data import, the data size of a single row after encoding must not exceed 32 MiB. +- When the Global Sort feature is used for data import, if the target TiDB cluster is deleted before the import task is completed, temporary data used for global sorting might remain on Amazon S3. In this case, you need to delete the residual data manually to avoid increasing S3 storage costs. +- Ensure that the data to be imported does not contain any records with primary key or non-null unique index conflicts. Otherwise, the conflicts can result in import task failures. +- If an `IMPORT INTO` task scheduled by the Distributed eXecution Framework (DXF) is already running, it cannot be scheduled to a new TiDB node. If the TiDB node that executes the data import task is restarted, it will no longer execute the data import task, but transfers the task to another TiDB node to continue executing. However, if the imported data is from a local file, the task will not be transferred to another TiDB node to continue executing. +- Known issue: the `IMPORT INTO` task might fail if the PD address in the TiDB node configuration file is inconsistent with the current PD topology of the cluster. This inconsistency can arise in situations such as that PD was scaled in previously, but the TiDB configuration file was not updated accordingly or the TiDB node was not restarted after the configuration file update. ## Prerequisites for import @@ -46,7 +43,7 @@ Before using `IMPORT INTO` to import data, make sure the following requirements - The target table to be imported is already created in TiDB and it is empty. - The target cluster has sufficient space to store the data to be imported. -- The [temporary directory](/tidb-configuration-file.md#temp-dir-new-in-v630) of the TiDB node connected to the current session has at least 90 GiB of available space. If [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) is enabled, also make sure that the temporary directory of each TiDB node in the cluster has sufficient disk space. +- For TiDB Self-Managed, the [temporary directory](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#temp-dir-new-in-v630) of the TiDB node connected to the current session has at least 90 GiB of available space. If [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) is enabled, also make sure that the temporary directory of each TiDB node in the cluster has sufficient disk space. ## Required privileges @@ -68,7 +65,7 @@ SetItem ::= ColumnName '=' Expr Format ::= - 'CSV' | 'SQL' | 'PARQUET' + 'FORMAT' ('"CSV"' | '"SQL"' | '"PARQUET"') WithOptions ::= 'WITH' OptionItem (',' OptionItem)* @@ -94,9 +91,10 @@ In the left side of the `SET` expression, you can only reference a column name t ### fileLocation -It specifies the storage location of the data file, which can be an Amazon S3, GCS, or Azure Blob Storage URI path, or a TiDB local file path. +It specifies the storage location of the data file, which can be an Amazon S3 or GCS URI path, or a TiDB local file path. + +- Amazon S3 or GCS URI path: for URI configuration details, see [URI Formats of External Storage Services](/external-storage-uri.md). -- Amazon S3, GCS, or Azure Blob Storage URI path: for URI configuration details, see [URI Formats of External Storage Services](/external-storage-uri.md). - TiDB local file path: it must be an absolute path, and the file extension must be `.csv`, `.sql`, or `.parquet`. Make sure that the files corresponding to this path are stored on the TiDB node connected by the current user, and the user has the `FILE` privilege. > **Note:** @@ -130,14 +128,14 @@ The supported options are described as follows: | `FIELDS_DEFINED_NULL_BY=''` | CSV | Specifies the value that represents `NULL` in the fields. The default value is `\N`. | | `LINES_TERMINATED_BY=''` | CSV | Specifies the line terminator. By default, `IMPORT INTO` automatically identifies `\n`, `\r`, or `\r\n` as line terminators. If the line terminator is one of these three, you do not need to explicitly specify this option. | | `SKIP_ROWS=` | CSV | Specifies the number of rows to skip. The default value is `0`. You can use this option to skip the header in a CSV file. If you use a wildcard to specify the source files for import, this option applies to all source files that are matched by the wildcard in `fileLocation`. | -| `SPLIT_FILE` | CSV | Splits a single CSV file into multiple smaller chunks of around 256 MiB for parallel processing to improve import efficiency. This parameter only works for **non-compressed** CSV files and has the same usage restrictions as that of TiDB Lightning [`strict-format`](/tidb-lightning/tidb-lightning-data-source.md#strict-format). | -| `DISK_QUOTA=''` | All formats | Specifies the disk space threshold that can be used during data sorting. The default value is 80% of the disk space in the TiDB [temporary directory](/tidb-configuration-file.md#temp-dir-new-in-v630). If the total disk size cannot be obtained, the default value is 50 GiB. When specifying `DISK_QUOTA` explicitly, make sure that the value does not exceed 80% of the disk space in the TiDB temporary directory. | +| `SPLIT_FILE` | CSV | Splits a single CSV file into multiple smaller chunks of around 256 MiB for parallel processing to improve import efficiency. This parameter only works for **non-compressed** CSV files and has the same usage restrictions as that of TiDB Lightning [`strict-format`](https://docs.pingcap.com/tidb/stable/tidb-lightning-data-source#strict-format). Note that you need to explicitly specify `LINES_TERMINATED_BY` for this option. | +| `DISK_QUOTA=''` | All formats | Specifies the disk space threshold that can be used during data sorting. The default value is 80% of the disk space in the TiDB [temporary directory](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#temp-dir-new-in-v630). If the total disk size cannot be obtained, the default value is 50 GiB. When specifying `DISK_QUOTA` explicitly, make sure that the value does not exceed 80% of the disk space in the TiDB temporary directory. | | `DISABLE_TIKV_IMPORT_MODE` | All formats | Specifies whether to disable switching TiKV to import mode during the import process. By default, switching TiKV to import mode is not disabled. If there are ongoing read-write operations in the cluster, you can enable this option to avoid impact from the import process. | | `THREAD=` | All formats | Specifies the concurrency for import. The default value is 50% of the CPU cores, with a minimum value of 1. You can explicitly specify this option to control the resource usage, but make sure that the value does not exceed the number of CPU cores. To import data into a new cluster without any data, it is recommended to increase this concurrency appropriately to improve import performance. If the target cluster is already used in a production environment, it is recommended to adjust this concurrency according to your application requirements. | | `MAX_WRITE_SPEED=''` | All formats | Controls the write speed to a TiKV node. By default, there is no speed limit. For example, you can specify this option as `1MiB` to limit the write speed to 1 MiB/s. | | `CHECKSUM_TABLE=''` | All formats | Configures whether to perform a checksum check on the target table after the import to validate the import integrity. The supported values include `"required"` (default), `"optional"`, and `"off"`. `"required"` means performing a checksum check after the import. If the checksum check fails, TiDB will return an error and the import will exit. `"optional"` means performing a checksum check after the import. If an error occurs, TiDB will return a warning and ignore the error. `"off"` means not performing a checksum check after the import. | | `DETACHED` | All Formats | Controls whether to execute `IMPORT INTO` asynchronously. When this option is enabled, executing `IMPORT INTO` immediately returns the information of the import job (such as the `Job_ID`), and the job is executed asynchronously in the backend. | -| `CLOUD_STORAGE_URI` | All formats | Specifies the target address where encoded KV data for [global sorting](#global-sorting) is stored. When `CLOUD_STORAGE_URI` is not specified, `IMPORT INTO` determines whether to use global sorting based on the value of the system variable [`tidb_cloud_storage_uri`](/system-variables.md#tidb_cloud_storage_uri-new-in-v740). If this system variable specifies a target storage address, `IMPORT INTO` uses this address for global sorting. When `CLOUD_STORAGE_URI` is specified with a non-empty value, `IMPORT INTO` uses that value as the target storage address. When `CLOUD_STORAGE_URI` is specified with an empty value, local sorting is enforced. Currently, the target storage address only supports S3. For details about the URI configuration, see [Amazon S3 URI format](/external-storage-uri.md#amazon-s3-uri-format). When this feature is used, all TiDB nodes must have read and write access for the target S3 bucket. | +| `CLOUD_STORAGE_URI` | All formats | Specifies the target address where encoded KV data for [Global Sort](/tidb-global-sort.md) is stored. When `CLOUD_STORAGE_URI` is not specified, `IMPORT INTO` determines whether to use Global Sort based on the value of the system variable [`tidb_cloud_storage_uri`](/system-variables.md#tidb_cloud_storage_uri-new-in-v740). If this system variable specifies a target storage address, `IMPORT INTO` uses this address for Global Sort. When `CLOUD_STORAGE_URI` is specified with a non-empty value, `IMPORT INTO` uses that value as the target storage address. When `CLOUD_STORAGE_URI` is specified with an empty value, local sorting is enforced. Currently, the target storage address only supports S3. For details about the URI configuration, see [Amazon S3 URI format](/external-storage-uri.md#amazon-s3-uri-format). When this feature is used, all TiDB nodes must have read and write access for the target S3 bucket. | ## Compressed files @@ -151,9 +149,18 @@ The supported options are described as follows: > **Note:** > -> The Snappy compressed file must be in the [official Snappy format](https://github.com/google/snappy). Other variants of Snappy compression are not supported. +> - The Snappy compressed file must be in the [official Snappy format](https://github.com/google/snappy). Other variants of Snappy compression are not supported. +> - Because TiDB Lightning cannot concurrently decompress a single large compressed file, the size of the compressed file affects the import speed. It is recommended that a source file is no greater than 256 MiB after decompression. -## Global sorting +## Global Sort + +> **Warning:** +> +> The Global Sort feature is experimental. It is not recommended to use it in production environments. + +> **Note:** +> +> Global Sort is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. `IMPORT INTO` splits the data import job of a source data file into multiple sub-jobs, each sub-job independently encoding and sorting data before importing. If the encoded KV ranges of these sub-jobs have significant overlap (to learn how TiDB encodes data to KV, see [TiDB computing](/tidb-computing.md)), TiKV needs to keep compaction during import, leading to a decrease in import performance and stability. @@ -163,12 +170,19 @@ In the following scenarios, there can be significant overlap in KV ranges: - `IMPORT INTO` splits sub-jobs based on the traversal order of data files, usually sorted by file name in lexicographic order. - If the target table has many indexes, or the index column values are scattered in the data file, the index KV generated by the encoding of each sub-job will also overlap. -When [Backend task distributed execution framework](/tidb-distributed-execution-framework.md) is enabled, you can enable global sorting by specifying the `CLOUD_STORAGE_URI` option in the `IMPORT INTO` statement or by specifying the target storage address for encoded KV data using the system variable [`tidb_cloud_storage_uri`](/system-variables.md#tidb_cloud_storage_uri-new-in-v740). Note that currently, only S3 is supported as the global sorting storage address. When global sorting is enabled, `IMPORT INTO` writes encoded KV data to the cloud storage, performs global sorting in the cloud storage, and then parallelly imports the globally sorted index and table data into TiKV. This prevents problems caused by KV overlap and enhances import stability. +When the [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md) is enabled, you can enable [Global Sort](/tidb-global-sort.md) by specifying the `CLOUD_STORAGE_URI` option in the `IMPORT INTO` statement or by specifying the target storage address for encoded KV data using the system variable [`tidb_cloud_storage_uri`](/system-variables.md#tidb_cloud_storage_uri-new-in-v740). Note that currently, only S3 is supported as the Global Sort storage address. When Global Sort is enabled, `IMPORT INTO` writes encoded KV data to the cloud storage, performs Global Sort in the cloud storage, and then parallelly imports the globally sorted index and table data into TiKV. This prevents problems caused by KV overlap and enhances import stability. + +Global Sort consumes a significant amount of memory resources. Before the data import, it is recommended to configure the [`tidb_server_memory_limit_gc_trigger`](/system-variables.md#tidb_server_memory_limit_gc_trigger-new-in-v640) and [`tidb_server_memory_limit`](/system-variables.md#tidb_server_memory_limit-new-in-v640) variables, which avoids golang GC being frequently triggered and thus affecting the import efficiency. + +```sql +SET GLOBAL tidb_server_memory_limit_gc_trigger=0.99; +SET GLOBAL tidb_server_memory_limit='88%'; +``` > **Note:** > -> - If the KV range overlap in a source data file is low, enabling global sorting might decrease import performance. This is because when global sorting is enabled, TiDB needs to wait for the completion of local sorting in all sub-jobs before proceeding with the global sorting operations and subsequent import. -> - After an import job using global sorting completes, the files stored in the cloud storage for global sorting are cleaned up asynchronously in a background thread. +> - If the KV range overlap in a source data file is low, enabling Global Sort might decrease import performance. This is because when Global Sort is enabled, TiDB needs to wait for the completion of local sorting in all sub-jobs before proceeding with the Global Sort operations and subsequent import. +> - After an import job using Global Sort completes, the files stored in the cloud storage for Global Sort are cleaned up asynchronously in a background thread. ## Output @@ -240,7 +254,7 @@ Assume that there are three files named `file-01.csv`, `file-02.csv`, and `file- IMPORT INTO t FROM '/path/to/file-*.csv' ``` -### Import data files from Amazon S3, GCS, or Azure Blob Storage +### Import data files from Amazon S3 or GCS - Import data files from Amazon S3: @@ -254,13 +268,7 @@ IMPORT INTO t FROM '/path/to/file-*.csv' IMPORT INTO t FROM 'gs://import/test.csv?credentials-file=${credentials-file-path}'; ``` -- Import data files from Azure Blob Storage: - - ```sql - IMPORT INTO t FROM 'azure://import/test.csv?credentials-file=${credentials-file-path}'; - ``` - -For details about the URI path configuration for Amazon S3, GCS, or Azure Blob Storage, see [URI Formats of External Storage Services](/external-storage-uri.md). +For details about the URI path configuration for Amazon S3 or GCS, see [URI Formats of External Storage Services](/external-storage-uri.md). ### Calculate column values using SetClause diff --git a/sql-statements/sql-statement-insert.md b/sql-statements/sql-statement-insert.md index ef36bda9e9e1f..f497a8239aef8 100644 --- a/sql-statements/sql-statement-insert.md +++ b/sql-statements/sql-statement-insert.md @@ -1,7 +1,6 @@ --- title: INSERT | TiDB SQL Statement Reference summary: An overview of the usage of INSERT for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-insert/','/docs/dev/reference/sql/statements/insert/'] --- # INSERT @@ -42,6 +41,10 @@ OnDuplicateKeyUpdate ::= ( 'ON' 'DUPLICATE' 'KEY' 'UPDATE' AssignmentList )? ``` +> **Note:** +> +> Starting from v6.6.0, TiDB supports [Resource Control](/tidb-resource-control.md). You can use this feature to execute SQL statements with different priorities in different resource groups. By configuring proper quotas and priorities for these resource groups, you can gain better scheduling control for SQL statements with different priorities. When resource control is enabled, statement priority (`PriorityOpt`) will no longer take effect. It is recommended that you use [Resource Control](/tidb-resource-control.md) to manage resource usage for different SQL statements. + ## Examples ```sql diff --git a/sql-statements/sql-statement-kill.md b/sql-statements/sql-statement-kill.md index dba75b9f2a4f2..5b16f4166a26e 100644 --- a/sql-statements/sql-statement-kill.md +++ b/sql-statements/sql-statement-kill.md @@ -1,12 +1,11 @@ --- title: KILL summary: An overview of the usage of KILL for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-kill/','/docs/dev/reference/sql/statements/kill/'] --- # KILL -The `KILL` statement is used to terminate a connection in any TiDB instance in the current TiDB cluster. +The `KILL` statement is used to terminate a connection in any TiDB instance in the current TiDB cluster. Starting from TiDB v6.2.0, you can also use the `KILL` statement to terminate ongoing DDL jobs. ## Synopsis diff --git a/sql-statements/sql-statement-load-data.md b/sql-statements/sql-statement-load-data.md index 8d8f27ed01e1f..d13522f33760e 100644 --- a/sql-statements/sql-statement-load-data.md +++ b/sql-statements/sql-statement-load-data.md @@ -1,7 +1,6 @@ --- title: LOAD DATA | TiDB SQL Statement Reference summary: An overview of the usage of LOAD DATA for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-load-data/','/docs/dev/reference/sql/statements/load-data/','/tidb/dev/sql-statement-operate-load-data-job','/tidb/dev/sql-statement-show-load-data'] --- # LOAD DATA @@ -38,15 +37,15 @@ Fields ::= You can use `LOCAL` to specify data files on the client to be imported, where the file parameter must be the file system path on the client. -If you are using TiDB Cloud, to use the `LOAD DATA` statement to load local data files, you need to add the `--local-infile` option to the connection string when you connect to TiDB Cloud. +If you are using TiDB Cloud, to use the `LOAD DATA` statement to load local data files, you need to add the `--local-infile` option to the connection string when you connect to TiDB Cloud. -- The following is an example connection string for TiDB Serverless: +- The following is an example connection string for {{{ .starter }}}: ``` mysql --connect-timeout 15 -u '' -h -P 4000 -D test --ssl-mode=VERIFY_IDENTITY --ssl-ca=/etc/ssl/cert.pem -p --local-infile ``` -- The following is an example connection string for TiDB Dedicated: +- The following is an example connection string for TiDB Cloud Dedicated: ``` mysql --connect-timeout 15 --ssl-mode=VERIFY_IDENTITY --ssl-ca= --tls-version="TLSv1.2" -u root -h -P 4000 -D test -p --local-infile @@ -153,9 +152,10 @@ The syntax of the `LOAD DATA` statement is compatible with that of MySQL, except > **Note:** > -> - For versions earlier than TiDB v4.0.0, `LOAD DATA` commits every 20000 rows. -> - For versions from TiDB v4.0.0 to v6.6.0, TiDB commits all rows in one transaction by default. -> - After upgrading from TiDB v4.0.0 or earlier versions, `ERROR 8004 (HY000) at line 1: Transaction is too large, size: 100000058` might occur. The recommended way to resolve this error is to increase the [`txn-total-size-limit`](/tidb-configuration-file.md#txn-total-size-limit) value in your `tidb.toml` file. If you are unable to increase this limit, you can also restore the behavior before the upgrade by setting [`tidb_dml_batch_size`](/system-variables.md#tidb_dml_batch_size) to `20000`. Note that starting from v7.0.0, `tidb_dml_batch_size` no longer takes effect on the `LOAD DATA` statement. +> - For versions earlier than TiDB v4.0.0, `LOAD DATA` commits every 20000 rows, which cannot be configured. +> - For versions from TiDB v4.0.0 to v6.6.0, TiDB commits all rows in one transaction by default. But if you need the `LOAD DATA` statement to commit every fixed number of rows, you can set [`tidb_dml_batch_size`](/system-variables.md#tidb_dml_batch_size) to the desired number of rows. +> - Starting from TiDB v7.0.0, `tidb_dml_batch_size` no longer takes effect on `LOAD DATA`, and TiDB commits all rows in one transaction. +> - After upgrading from TiDB v4.0.0 or earlier versions, `ERROR 8004 (HY000) at line 1: Transaction is too large, size: 100000058` might occur. The recommended way to resolve this error is to increase the [`txn-total-size-limit`](/tidb-configuration-file.md#txn-total-size-limit) value in your `tidb.toml` file. > - No matter how many rows are committed in a transaction, `LOAD DATA` is not rolled back by the [`ROLLBACK`](/sql-statements/sql-statement-rollback.md) statement in an explicit transaction. > - The `LOAD DATA` statement is always executed in optimistic transaction mode, regardless of the TiDB transaction mode configuration. @@ -165,10 +165,10 @@ The syntax of the `LOAD DATA` statement is compatible with that of MySQL, except > **Note:** > -> - For versions earlier than TiDB v4.0.0, `LOAD DATA` commits every 20000 rows. -> - For versions from TiDB v4.0.0 to v6.6.0, TiDB commits all rows in one transaction by default. -> - Starting from TiDB v7.0.0, the number of rows to be committed in a batch is controlled by the `WITH batch_size=` parameter of the `LOAD DATA` statement, which defaults to 1000 rows per commit. -> - After upgrading from TiDB v4.0.0 or earlier versions, `ERROR 8004 (HY000) at line 1: Transaction is too large, size: 100000058` might occur. To resolve this error, you can restore the behavior before the upgrade by setting [`tidb_dml_batch_size`](/system-variables.md#tidb_dml_batch_size) to `20000`. +> - For versions earlier than TiDB v4.0.0, `LOAD DATA` commits every 20000 rows, which cannot be configured. +> - For versions from TiDB v4.0.0 to v6.6.0, TiDB commits all rows in one transaction by default. But if you need the `LOAD DATA` statement to commit every fixed number of rows, you can set [`tidb_dml_batch_size`](/system-variables.md#tidb_dml_batch_size) to the desired number of rows. +> - Starting from v7.0.0, `tidb_dml_batch_size` no longer takes effect on `LOAD DATA`, and TiDB commits all rows in one transaction. +> - After upgrading from TiDB v4.0.0 or earlier versions, `ERROR 8004 (HY000) at line 1: Transaction is too large, size: 100000058` might occur. To resolve this error, you can contact [TiDB Cloud Support](https://docs.pingcap.com/tidbcloud/tidb-cloud-support) to increase the [`txn-total-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-total-size-limit) value. > - No matter how many rows are committed in a transaction, `LOAD DATA` is not rolled back by the [`ROLLBACK`](/sql-statements/sql-statement-rollback.md) statement in an explicit transaction. > - The `LOAD DATA` statement is always executed in optimistic transaction mode, regardless of the TiDB transaction mode configuration. diff --git a/sql-statements/sql-statement-load-stats.md b/sql-statements/sql-statement-load-stats.md index d3d8edea449e8..e5f436d052e0e 100644 --- a/sql-statements/sql-statement-load-stats.md +++ b/sql-statements/sql-statement-load-stats.md @@ -1,7 +1,6 @@ --- title: LOAD STATS summary: An overview of the usage of LOAD STATS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-load-stats/'] --- # LOAD STATS @@ -10,7 +9,7 @@ The `LOAD STATS` statement is used to load the statistics into TiDB. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-lock-stats.md b/sql-statements/sql-statement-lock-stats.md index a924a0b2c1a36..25057e7ba9476 100644 --- a/sql-statements/sql-statement-lock-stats.md +++ b/sql-statements/sql-statement-lock-stats.md @@ -7,10 +7,6 @@ summary: An overview of the usage of LOCK STATS for the TiDB database. `LOCK STATS` is used to lock the statistics of tables or partitions. When the statistics is locked, TiDB does not automatically update the statistics of the table or partition. For details on the behavior, see [Behaviors of locking statistics](/statistics.md#behaviors-of-locking-statistics). -> **Warning:** -> -> Locking statistics is an experimental feature for the current version. It is not recommended to use it in the production environment. - ## Synopsis ```ebnf+diagram diff --git a/sql-statements/sql-statement-lock-tables-and-unlock-tables.md b/sql-statements/sql-statement-lock-tables-and-unlock-tables.md index 1d1c0c306e365..da0bcc99ec81f 100644 --- a/sql-statements/sql-statement-lock-tables-and-unlock-tables.md +++ b/sql-statements/sql-statement-lock-tables-and-unlock-tables.md @@ -21,8 +21,9 @@ A table lock protects against reads or writes by other sessions. A session that > > The table locks feature is disabled by default. > -> - For TiDB Self-Hosted, to enable the table locks feature, you need to set [`enable-table-lock`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#enable-table-lock-new-in-v400) to `true` in the configuration files of all TiDB instances. -> - For TiDB Cloud, to enable the table locks feature, you need to contact [TiDB Cloud Support](https://docs.pingcap.com/tidbcloud/tidb-cloud-support) to set [`enable-table-lock`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#enable-table-lock-new-in-v400) to `true`. +> - For TiDB Self-Managed, to enable the table locks feature, you need to set [`enable-table-lock`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#enable-table-lock-new-in-v400) to `true` in the configuration files of all TiDB instances. +> - For TiDB Cloud Dedicated, to enable the table locks feature, you need to contact [TiDB Cloud Support](https://docs.pingcap.com/tidbcloud/tidb-cloud-support) to set [`enable-table-lock`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#enable-table-lock-new-in-v400) to `true`. +> - For [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential), setting [`enable-table-lock`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#enable-table-lock-new-in-v400) to `true` is not supported. ## Synopsis diff --git a/sql-statements/sql-statement-modify-column.md b/sql-statements/sql-statement-modify-column.md index 12748bfe00de6..2a51cf0ba3af2 100644 --- a/sql-statements/sql-statement-modify-column.md +++ b/sql-statements/sql-statement-modify-column.md @@ -1,7 +1,6 @@ --- title: MODIFY COLUMN | TiDB SQL Statement Reference summary: An overview of the usage of MODIFY COLUMN for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-modify-column/','/docs/dev/reference/sql/statements/modify-column/'] --- # MODIFY COLUMN @@ -88,7 +87,7 @@ Query OK, 0 rows affected (0.09 sec) {{< copyable "sql" >}} ```sql -SHOW CREATE TABLE t1\G; +SHOW CREATE TABLE t1\G ``` ```sql @@ -138,7 +137,7 @@ Query OK, 0 rows affected (2.52 sec) {{< copyable "sql" >}} ```sql -SHOW CREATE TABLE t1\G; +SHOW CREATE TABLE t1\G ``` ```sql @@ -205,7 +204,7 @@ CREATE TABLE `t1` ( ERROR 8200 (HY000): Unsupported modify column: table is partition table ``` -* Does not support modifying some data types (for example, some TIME types, Bit, Set, Enum, JSON) are not supported due to some compatibility issues of the `cast` function's behavior between TiDB and MySQL. +* Does not support modifying from some data types (for example, some TIME types, BIT, SET, ENUM, JSON) to some other types due to some compatibility issues of the `cast` function's behavior between TiDB and MySQL. ```sql CREATE TABLE t (a DECIMAL(13, 7)); diff --git a/sql-statements/sql-statement-prepare.md b/sql-statements/sql-statement-prepare.md index d13088ae7907e..a8690a344408a 100644 --- a/sql-statements/sql-statement-prepare.md +++ b/sql-statements/sql-statement-prepare.md @@ -1,7 +1,6 @@ --- title: PREPARE | TiDB SQL Statement Reference summary: An overview of the usage of PREPARE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-prepare/','/docs/dev/reference/sql/statements/prepare/'] --- # PREPARE diff --git a/sql-statements/sql-statement-query-watch.md b/sql-statements/sql-statement-query-watch.md index a635d54c231a6..fb7a63fd3ebe2 100644 --- a/sql-statements/sql-statement-query-watch.md +++ b/sql-statements/sql-statement-query-watch.md @@ -11,6 +11,10 @@ The `QUERY WATCH` statement is used to manually manage the watch list of runaway > > This feature is experimental. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. +> **Note:** +> +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. + ## Synopsis ```ebnf+diagram diff --git a/sql-statements/sql-statement-recover-table.md b/sql-statements/sql-statement-recover-table.md index 87f12aa90e753..e5f46d6d3315a 100644 --- a/sql-statements/sql-statement-recover-table.md +++ b/sql-statements/sql-statement-recover-table.md @@ -1,7 +1,6 @@ --- title: RECOVER TABLE summary: An overview of the usage of RECOVER TABLE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-recover-table/','/docs/dev/reference/sql/statements/recover-table/'] --- # RECOVER TABLE diff --git a/sql-statements/sql-statement-rename-index.md b/sql-statements/sql-statement-rename-index.md index 6d1c08d29edcc..9d419ab7c0238 100644 --- a/sql-statements/sql-statement-rename-index.md +++ b/sql-statements/sql-statement-rename-index.md @@ -1,7 +1,6 @@ --- title: RENAME INDEX | TiDB SQL Statement Reference summary: An overview of the usage of RENAME INDEX for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-rename-index/','/docs/dev/reference/sql/statements/rename-index/'] --- # RENAME INDEX @@ -59,5 +58,5 @@ The `RENAME INDEX` statement in TiDB is fully compatible with MySQL. If you find * [SHOW CREATE TABLE](/sql-statements/sql-statement-show-create-table.md) * [CREATE INDEX](/sql-statements/sql-statement-create-index.md) * [DROP INDEX](/sql-statements/sql-statement-drop-index.md) -* [SHOW INDEX](/sql-statements/sql-statement-show-index.md) +* [SHOW INDEXES](/sql-statements/sql-statement-show-indexes.md) * [ALTER INDEX](/sql-statements/sql-statement-alter-index.md) diff --git a/sql-statements/sql-statement-rename-table.md b/sql-statements/sql-statement-rename-table.md index 6f03a06f4ca91..d200beba61ffd 100644 --- a/sql-statements/sql-statement-rename-table.md +++ b/sql-statements/sql-statement-rename-table.md @@ -1,12 +1,11 @@ --- title: RENAME TABLE | TiDB SQL Statement Reference summary: An overview of the usage of RENAME TABLE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-rename-table/','/docs/dev/reference/sql/statements/rename-table/'] --- # RENAME TABLE -This statement renames an existing table to a new name. +This statement is used to rename existing tables and views, supporting renaming multiple tables at once and renaming across databases. ## Synopsis @@ -21,21 +20,39 @@ TableToTable ::= ## Examples ```sql -mysql> CREATE TABLE t1 (a int); +CREATE TABLE t1 (a int); +``` + +``` Query OK, 0 rows affected (0.12 sec) +``` + +```sql +SHOW TABLES; +``` -mysql> SHOW TABLES; +``` +----------------+ | Tables_in_test | +----------------+ | t1 | +----------------+ 1 row in set (0.00 sec) +``` + +```sql +RENAME TABLE t1 TO t2; +``` -mysql> RENAME TABLE t1 TO t2; +``` Query OK, 0 rows affected (0.08 sec) +``` + +```sql +SHOW TABLES; +``` -mysql> SHOW TABLES; +``` +----------------+ | Tables_in_test | +----------------+ @@ -44,6 +61,103 @@ mysql> SHOW TABLES; 1 row in set (0.00 sec) ``` +The following example demonstrates how to rename multiple tables across databases, assuming that the databases `db1`, `db2`, `db3`, and `db4` already exist, and that the tables `db1.t1` and `db3.t3` already exist: + +```sql +RENAME TABLE db1.t1 To db2.t2, db3.t3 To db4.t4; +``` + +``` +Query OK, 0 rows affected (0.08 sec) +``` + +```sql +USE db1; SHOW TABLES; +``` + +``` +Database changed +Empty set (0.00 sec) +``` + +```sql +USE db2; SHOW TABLES; +``` + +``` +Database changed ++---------------+ +| Tables_in_db2 | ++---------------+ +| t2 | ++---------------+ +1 row in set (0.00 sec) +``` + +```sql +USE db3; SHOW TABLES; +``` + +``` +Database changed +Empty set (0.00 sec) +``` + +```sql +USE db4; SHOW TABLES; +``` + +``` +Database changed ++---------------+ +| Tables_in_db4 | ++---------------+ +| t4 | ++---------------+ +1 row in set (0.00 sec) +``` + +The atomic rename can be used to swap out a table without having any moment in which the table does not exist. + +```sql +CREATE TABLE t1(id int PRIMARY KEY); +``` + +``` +Query OK, 0 rows affected (0.04 sec) +``` + +```sql +CREATE TABLE t1_new(id int PRIMARY KEY, n CHAR(0)); +```` + +``` +Query OK, 0 rows affected (0.04 sec) +``` + +```sql +RENAME TABLE t1 TO t1_old, t1_new TO t1; +``` + +``` +Query OK, 0 rows affected (0.07 sec) +``` + +```sql +SHOW CREATE TABLE t1\G +``` + +``` +*************************** 1. row *************************** + Table: t1 +Create Table: CREATE TABLE `t1` ( + `id` int NOT NULL, + `n` char(0) DEFAULT NULL, + PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin +1 row in set (0.00 sec) +``` + ## MySQL compatibility The `RENAME TABLE` statement in TiDB is fully compatible with MySQL. If you find any compatibility differences, [report a bug](https://docs.pingcap.com/tidb/stable/support). diff --git a/sql-statements/sql-statement-rename-user.md b/sql-statements/sql-statement-rename-user.md index 3cd298e825c98..1ad0c8a68fc98 100644 --- a/sql-statements/sql-statement-rename-user.md +++ b/sql-statements/sql-statement-rename-user.md @@ -5,7 +5,7 @@ summary: An overview of the usage of RENAME USER for the TiDB database. # RENAME USER -`RANAME USER` is used to rename an existing user. +`RENAME USER` is used to rename an existing user. ## Synopsis diff --git a/sql-statements/sql-statement-replace.md b/sql-statements/sql-statement-replace.md index bb5912d8d2715..d44db793a3af0 100644 --- a/sql-statements/sql-statement-replace.md +++ b/sql-statements/sql-statement-replace.md @@ -1,7 +1,6 @@ --- title: REPLACE | TiDB SQL Statement Reference summary: An overview of the usage of REPLACE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-replace/','/docs/dev/reference/sql/statements/replace/'] --- # REPLACE @@ -33,6 +32,10 @@ InsertValues ::= | 'SET' ColumnSetValue? ( ',' ColumnSetValue )* ``` +> **Note:** +> +> Starting from v6.6.0, TiDB supports [Resource Control](/tidb-resource-control.md). You can use this feature to execute SQL statements with different priorities in different resource groups. By configuring proper quotas and priorities for these resource groups, you can gain better scheduling control for SQL statements with different priorities. When resource control is enabled, statement priority (`PriorityOpt`) will no longer take effect. It is recommended that you use [Resource Control](/tidb-resource-control.md) to manage resource usage for different SQL statements. + ## Examples ```sql diff --git a/sql-statements/sql-statement-restore.md b/sql-statements/sql-statement-restore.md index 1b817a911c019..a01f699db7470 100644 --- a/sql-statements/sql-statement-restore.md +++ b/sql-statements/sql-statement-restore.md @@ -1,16 +1,16 @@ --- title: RESTORE | TiDB SQL Statement Reference summary: An overview of the usage of RESTORE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-restore/'] --- # RESTORE This statement performs a distributed restore from a backup archive previously produced by a [`BACKUP` statement](/sql-statements/sql-statement-backup.md). -> **Note:** +> **Warning:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> - This feature is experimental. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. +> - This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. The `RESTORE` statement uses the same engine as the [BR tool](https://docs.pingcap.com/tidb/stable/backup-and-restore-overview), except that the restore process is driven by TiDB itself rather than a separate BR tool. All benefits and caveats of BR also apply here. In particular, **`RESTORE` is currently not ACID-compliant**. Before running `RESTORE`, ensure that the following requirements are met: diff --git a/sql-statements/sql-statement-revoke-privileges.md b/sql-statements/sql-statement-revoke-privileges.md index dc60a2658b236..9fb6f6681e9f5 100644 --- a/sql-statements/sql-statement-revoke-privileges.md +++ b/sql-statements/sql-statement-revoke-privileges.md @@ -1,7 +1,6 @@ --- title: REVOKE | TiDB SQL Statement Reference summary: An overview of the usage of REVOKE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-revoke-privileges/','/docs/dev/reference/sql/statements/revoke-privileges/'] --- # `REVOKE ` @@ -11,8 +10,8 @@ This statement removes privileges from an existing user. Executing this statemen ## Synopsis ```ebnf+diagram -GrantStmt ::= - 'GRANT' PrivElemList 'ON' ObjectType PrivLevel 'TO' UserSpecList RequireClauseOpt WithGrantOptionOpt +RevokeStmt ::= + 'REVOKE' PrivElemList 'ON' ObjectType PrivLevel 'FROM' UserSpecList PrivElemList ::= PrivElem ( ',' PrivElem )* diff --git a/sql-statements/sql-statement-rollback.md b/sql-statements/sql-statement-rollback.md index 5a08335a17243..3da389ce3c5a1 100644 --- a/sql-statements/sql-statement-rollback.md +++ b/sql-statements/sql-statement-rollback.md @@ -1,7 +1,6 @@ --- title: ROLLBACK | TiDB SQL Statement Reference summary: An overview of the usage of ROLLBACK for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-rollback/','/docs/dev/reference/sql/statements/rollback/'] --- # ROLLBACK diff --git a/sql-statements/sql-statement-select.md b/sql-statements/sql-statement-select.md index 63455b8ddd392..15aeca289b921 100644 --- a/sql-statements/sql-statement-select.md +++ b/sql-statements/sql-statement-select.md @@ -1,7 +1,6 @@ --- title: SELECT | TiDB SQL Statement Reference summary: An overview of the usage of SELECT for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-select/','/docs/dev/reference/sql/statements/select/'] --- # SELECT @@ -14,10 +13,6 @@ The `SELECT` statement is used to read data from TiDB. ![SelectStmt](/media/sqlgram/SelectStmt.png) -> **Note:** -> -> The `SELECT ... INTO OUTFILE` statement is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). - **FromDual:** ![FromDual](/media/sqlgram/FromDual.png) @@ -116,8 +111,14 @@ TableSampleOpt ::= |`LOCK IN SHARE MODE` | To guarantee compatibility, TiDB parses these three modifiers, but will ignore them. | | `TABLESAMPLE` | To get a sample of rows from the table. | +> **Note:** +> +> Starting from v6.6.0, TiDB supports [Resource Control](/tidb-resource-control.md). You can use this feature to execute SQL statements with different priorities in different resource groups. By configuring proper quotas and priorities for these resource groups, you can gain better scheduling control for SQL statements with different priorities. When resource control is enabled, statement priority (`HIGH_PRIORITY`) will no longer take effect. It is recommended that you use [Resource Control](/tidb-resource-control.md) to manage resource usage for different SQL statements. + ## Examples +### SELECT + ```sql mysql> CREATE TABLE t1 (id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, c1 INT NOT NULL); Query OK, 0 rows affected (0.11 sec) @@ -159,11 +160,85 @@ mysql> SELECT AVG(s_quantity), COUNT(s_quantity) FROM stock; The above example uses data generated with `tiup bench tpcc prepare`. The first query shows the use of `TABLESAMPLE`. +### SELECT ... INTO OUTFILE + +The `SELECT ... INTO OUTFILE` statement is used to write the result of a query to a file. + +> **Note:** +> +> - This statement is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> - This statement does not support writing query results to any [external storages](https://docs.pingcap.com/tidb/stable/backup-and-restore-storages) such as Amazon S3 or GCS. + +In the statement, you can specify the format of the output file by using the following clauses: + +- `FIELDS TERMINATED BY`: specifies the field delimiter in the file. For example, you can specify it as `','` to output comma-separated values (CSV) or `'\t'` to output tab-separated values (TSV). +- `FIELDS ENCLOSED BY`: specifies the enclosing character that wraps around each field in the file. +- `LINES TERMINATED BY`: specifies the line terminator in the file, if you want to end a line with a certain character. + +Assume that there is a table `t` with three columns as follows: + +```sql +mysql> CREATE TABLE t (a INT, b VARCHAR(10), c DECIMAL(10,2)); +Query OK, 0 rows affected (0.02 sec) + +mysql> INSERT INTO t VALUES (1, 'a', 1.1), (2, 'b', 2.2), (3, 'c', 3.3); +Query OK, 3 rows affected (0.01 sec) +``` + +The following examples show how to use the `SELECT ... INTO OUTFILE` statement to write the query result to a file. + +**Example 1:** + +```sql +mysql> SELECT * FROM t INTO OUTFILE '/tmp/tmp_file1'; +Query OK, 3 rows affected (0.00 sec) +``` + +In this example, you can find the query result in `/tmp/tmp_file1` as follows: + +``` +1 a 1.10 +2 b 2.20 +3 c 3.30 +``` + +**Example 2:** + +```sql +mysql> SELECT * FROM t INTO OUTFILE '/tmp/tmp_file2' FIELDS TERMINATED BY ',' ENCLOSED BY '"'; +Query OK, 3 rows affected (0.00 sec) +``` + +In this example, you can find the query result in `/tmp/tmp_file2` as follows: + +``` +"1","a","1.10" +"2","b","2.20" +"3","c","3.30" +``` + +**Example 3:** + +```sql +mysql> SELECT * FROM t INTO OUTFILE '/tmp/tmp_file3' + -> FIELDS TERMINATED BY ',' ENCLOSED BY '\'' LINES TERMINATED BY '<<<\n'; +Query OK, 3 rows affected (0.00 sec) +``` + +In this example, you can find the query result in `/tmp/tmp_file3` as follows: + +``` +'1','a','1.10'<<< +'2','b','2.20'<<< +'3','c','3.30'<<< +``` + ## MySQL compatibility - The syntax `SELECT ... INTO @variable` is not supported. +- The syntax `SELECT ... INTO DUMPFILE` is not supported. - The syntax `SELECT .. GROUP BY expr` does not imply `GROUP BY expr ORDER BY expr` as it does in MySQL 5.7. TiDB instead matches the behavior of MySQL 8.0 and does not imply a default order. -- The syntax `SELECT ... TABLESAMPLE ...` is a TiDB extension and not supported by MySQL. +- The syntax `SELECT ... TABLESAMPLE ...` is a TiDB extension designed for compatibility with other database systems and the [ISO/IEC 9075-2](https://standards.iso.org/iso-iec/9075/-2/ed-6/en/) standard, but currently it is not supported by MySQL. ## See also diff --git a/sql-statements/sql-statement-set-default-role.md b/sql-statements/sql-statement-set-default-role.md index 6686fcc5bf066..aa90b5cdcd0b1 100644 --- a/sql-statements/sql-statement-set-default-role.md +++ b/sql-statements/sql-statement-set-default-role.md @@ -9,21 +9,10 @@ This statement sets a specific role to be applied to a user by default. Thus, th ## Synopsis -**SetDefaultRoleStmt:** - -![SetDefaultRoleStmt](/media/sqlgram/SetDefaultRoleStmt.png) - -**SetDefaultRoleOpt:** - -![SetDefaultRoleOpt](/media/sqlgram/SetDefaultRoleOpt.png) - -**RolenameList:** - -![RolenameList](/media/sqlgram/RolenameList.png) - -**UsernameList:** - -![UsernameList](/media/sqlgram/UsernameList.png) +```ebnf+diagram +SetDefaultRoleStmt ::= + "SET" "DEFAULT" "ROLE" ( "NONE" | "ALL" | Rolename ("," Rolename)* ) "TO" Username ("," Username)* +``` ## Examples diff --git a/sql-statements/sql-statement-set-names.md b/sql-statements/sql-statement-set-names.md index 5c99b307cb9bc..f5d384846943f 100644 --- a/sql-statements/sql-statement-set-names.md +++ b/sql-statements/sql-statement-set-names.md @@ -1,7 +1,6 @@ --- title: SET [NAMES|CHARACTER SET] | TiDB SQL Statement Reference summary: An overview of the usage of SET [NAMES|CHARACTER SET] for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-set-names/','/docs/dev/reference/sql/statements/set-names/'] --- # SET [NAMES|CHARACTER SET] diff --git a/sql-statements/sql-statement-set-password.md b/sql-statements/sql-statement-set-password.md index 039d162437ba2..9d861fc0b4762 100644 --- a/sql-statements/sql-statement-set-password.md +++ b/sql-statements/sql-statement-set-password.md @@ -1,7 +1,6 @@ --- title: SET PASSWORD | TiDB SQL Statement Reference summary: An overview of the usage of SET PASSWORD for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-set-password/','/docs/dev/reference/sql/statements/set-password/'] --- # SET PASSWORD @@ -10,9 +9,10 @@ This statement changes the user password for a user account in the TiDB system d ## Synopsis -**SetStmt:** - -![SetStmt](/media/sqlgram/SetStmt.png) +```ebnf+diagram +SetPasswordStmt ::= + "SET" "PASSWORD" ( "FOR" Username )? "=" ( stringLit | "PASSWORD" "(" stringLit ")" ) +``` ## Examples diff --git a/sql-statements/sql-statement-set-resource-group.md b/sql-statements/sql-statement-set-resource-group.md index 6e43d00501965..88141d1993a4f 100644 --- a/sql-statements/sql-statement-set-resource-group.md +++ b/sql-statements/sql-statement-set-resource-group.md @@ -9,7 +9,7 @@ summary: An overview of the usage of SET RESOURCE GROUP in the TiDB database. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-set-role.md b/sql-statements/sql-statement-set-role.md index 703b8e62db0a1..dfacc4f924169 100644 --- a/sql-statements/sql-statement-set-role.md +++ b/sql-statements/sql-statement-set-role.md @@ -1,7 +1,6 @@ --- title: SET ROLE | TiDB SQL Statement Reference summary: An overview of the usage of SET ROLE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-set-role/'] --- # SET ROLE diff --git a/sql-statements/sql-statement-set-transaction.md b/sql-statements/sql-statement-set-transaction.md index 56c6354d533bb..acb08f235404a 100644 --- a/sql-statements/sql-statement-set-transaction.md +++ b/sql-statements/sql-statement-set-transaction.md @@ -1,7 +1,6 @@ --- title: SET TRANSACTION | TiDB SQL Statement Reference summary: An overview of the usage of SET TRANSACTION for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-set-transaction/','/docs/dev/reference/sql/statements/set-transaction/'] --- # SET TRANSACTION diff --git a/sql-statements/sql-statement-set-variable.md b/sql-statements/sql-statement-set-variable.md index 6fb9326754b5d..f45739db6ffc5 100644 --- a/sql-statements/sql-statement-set-variable.md +++ b/sql-statements/sql-statement-set-variable.md @@ -1,7 +1,6 @@ --- title: SET [GLOBAL|SESSION] | TiDB SQL Statement Reference summary: An overview of the usage of SET [GLOBAL|SESSION] for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-set-variable/','/docs/dev/reference/sql/statements/set-variable/'] --- # `SET [GLOBAL|SESSION] ` diff --git a/sql-statements/sql-statement-show-analyze-status.md b/sql-statements/sql-statement-show-analyze-status.md index 087a6e8819176..2f127c1f9617e 100644 --- a/sql-statements/sql-statement-show-analyze-status.md +++ b/sql-statements/sql-statement-show-analyze-status.md @@ -1,7 +1,6 @@ --- title: SHOW ANALYZE STATUS summary: An overview of the usage of SHOW ANALYZE STATUS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-analyze-status/'] --- # SHOW ANALYZE STATUS diff --git a/sql-statements/sql-statement-show-backups.md b/sql-statements/sql-statement-show-backups.md index e093b7a7445e2..eed71c1e541b4 100644 --- a/sql-statements/sql-statement-show-backups.md +++ b/sql-statements/sql-statement-show-backups.md @@ -1,7 +1,6 @@ --- title: SHOW [BACKUPS|RESTORES] | TiDB SQL Statement Reference summary: An overview of the usage of SHOW [BACKUPS|RESTORES] for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-backups/'] --- # SHOW [BACKUPS|RESTORES] @@ -14,7 +13,7 @@ Use `SHOW BACKUPS` to query `BACKUP` tasks and use `SHOW RESTORES` to query `RES > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. Backups and restores that were started with the `br` commandline tool are not shown. diff --git a/sql-statements/sql-statement-show-bindings.md b/sql-statements/sql-statement-show-bindings.md index 53d2dcb001e76..b31b177c0a12c 100644 --- a/sql-statements/sql-statement-show-bindings.md +++ b/sql-statements/sql-statement-show-bindings.md @@ -1,7 +1,6 @@ --- title: SHOW [GLOBAL|SESSION] BINDINGS summary: Use of SHOW BINDINGS binding in TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-bindings/'] --- # SHOW [GLOBAL|SESSION] BINDINGS @@ -10,30 +9,17 @@ The `SHOW BINDINGS` statement is used to display information about created SQL b ## Synopsis -**ShowStmt:** +```ebnf+diagram +ShowBindingsStmt ::= + "SHOW" ("GLOBAL" | "SESSION")? "BINDINGS" ShowLikeOrWhere? -![ShowStmt](/media/sqlgram/ShowStmt.png) - -**ShowTargetFilterable:** - -![ShowTargetFilterable](/media/sqlgram/ShowTargetFilterable.png) - -**GlobalScope:** - -![GlobalScope](/media/sqlgram/GlobalScope.png) - -**ShowLikeOrWhereOpt** - -![ShowLikeOrWhereOpt](/media/sqlgram/ShowLikeOrWhereOpt.png) +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` ## Syntax description -{{< copyable "sql" >}} - -```sql -SHOW [GLOBAL | SESSION] BINDINGS [ShowLikeOrWhereOpt]; -``` - This statement outputs the execution plan bindings at the GLOBAL or SESSION level. The default scope is SESSION. Currently `SHOW BINDINGS` outputs eight columns, as shown below: | Column Name | Description | @@ -50,8 +36,6 @@ This statement outputs the execution plan bindings at the GLOBAL or SESSION leve ## Examples -{{< copyable "sql" >}} - ```sql mysql> CREATE TABLE t1 ( id INT NOT NULL PRIMARY KEY auto_increment, diff --git a/sql-statements/sql-statement-show-builtins.md b/sql-statements/sql-statement-show-builtins.md index df6186adb3c42..a3dcc803d4571 100644 --- a/sql-statements/sql-statement-show-builtins.md +++ b/sql-statements/sql-statement-show-builtins.md @@ -1,7 +1,6 @@ --- title: SHOW BUILTINS summary: The usage of SHOW BUILTINS in TiDB. -aliases: ['/docs/dev/sql-statements/sql-statement-show-builtins/'] --- # SHOW BUILTINS @@ -10,14 +9,13 @@ aliases: ['/docs/dev/sql-statements/sql-statement-show-builtins/'] ## Synopsis -**ShowBuiltinsStmt:** - -![ShowBuiltinsStmt](/media/sqlgram/ShowBuiltinsStmt.png) +```ebnf+diagram +ShowBuiltinsStmt ::= + "SHOW" "BUILTINS" +``` ## Examples -{{< copyable "sql" >}} - ```sql SHOW BUILTINS; ``` diff --git a/sql-statements/sql-statement-show-character-set.md b/sql-statements/sql-statement-show-character-set.md index 478ed1fa5b98f..8377ea122a9d7 100644 --- a/sql-statements/sql-statement-show-character-set.md +++ b/sql-statements/sql-statement-show-character-set.md @@ -1,7 +1,6 @@ --- title: SHOW CHARACTER SET | TiDB SQL Statement Reference summary: An overview of the usage of SHOW CHARACTER SET for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-character-set/','/docs/dev/reference/sql/statements/show-character-set/'] --- # SHOW CHARACTER SET @@ -10,28 +9,61 @@ This statement provides a static list of available character sets in TiDB. The o ## Synopsis -**ShowCharsetStmt:** +```ebnf+diagram +ShowCharsetStmt ::= + "SHOW" ( ("CHARACTER" | "CHAR") "SET" | "CHARSET" ) ShowLikeOrWhere? -![ShowCharsetStmt](/media/sqlgram/ShowCharsetStmt.png) +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` -**CharsetKw:** +## Examples -![CharsetKw](/media/sqlgram/CharsetKw.png) +```sql +SHOW CHARACTER SET; +``` -## Examples +``` ++---------+-------------------------------------+-------------------+--------+ +| Charset | Description | Default collation | Maxlen | ++---------+-------------------------------------+-------------------+--------+ +| ascii | US ASCII | ascii_bin | 1 | +| binary | binary | binary | 1 | +| gbk | Chinese Internal Code Specification | gbk_chinese_ci | 2 | +| latin1 | Latin1 | latin1_bin | 1 | +| utf8 | UTF-8 Unicode | utf8_bin | 3 | +| utf8mb4 | UTF-8 Unicode | utf8mb4_bin | 4 | ++---------+-------------------------------------+-------------------+--------+ +6 rows in set (0.00 sec) +``` + +```sql +SHOW CHARACTER SET LIKE 'utf8%'; +``` + +``` ++---------+---------------+-------------------+--------+ +| Charset | Description | Default collation | Maxlen | ++---------+---------------+-------------------+--------+ +| utf8 | UTF-8 Unicode | utf8_bin | 3 | +| utf8mb4 | UTF-8 Unicode | utf8mb4_bin | 4 | ++---------+---------------+-------------------+--------+ +2 rows in set (0.00 sec) +``` ```sql -mysql> SHOW CHARACTER SET; +SHOW CHARACTER SET WHERE Description='UTF-8 Unicode'; +``` + +``` +---------+---------------+-------------------+--------+ | Charset | Description | Default collation | Maxlen | +---------+---------------+-------------------+--------+ | utf8 | UTF-8 Unicode | utf8_bin | 3 | | utf8mb4 | UTF-8 Unicode | utf8mb4_bin | 4 | -| ascii | US ASCII | ascii_bin | 1 | -| latin1 | Latin1 | latin1_bin | 1 | -| binary | binary | binary | 1 | +---------+---------------+-------------------+--------+ -5 rows in set (0.00 sec) +2 rows in set (0.00 sec) ``` ## MySQL compatibility diff --git a/sql-statements/sql-statement-show-collation.md b/sql-statements/sql-statement-show-collation.md index afcdb02bacfe3..075bb742b18ec 100644 --- a/sql-statements/sql-statement-show-collation.md +++ b/sql-statements/sql-statement-show-collation.md @@ -1,7 +1,6 @@ --- title: SHOW COLLATION | TiDB SQL Statement Reference summary: An overview of the usage of SHOW COLLATION for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-collation/','/docs/dev/reference/sql/statements/show-collation/'] --- # SHOW COLLATION @@ -14,16 +13,24 @@ This statement provides a static list of collations, and is included to provide ## Synopsis -**ShowCollationStmt:** +```ebnf+diagram +ShowCollationStmt ::= + "SHOW" "COLLATION" ShowLikeOrWhere? -![ShowCollationStmt](/media/sqlgram/ShowCollationStmt.png) +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` ## Examples When new collation framework is disabled, only binary collations are displayed. ```sql -mysql> SHOW COLLATION; +SHOW COLLATION; +``` + +``` +-------------+---------+------+---------+----------+---------+ | Collation | Charset | Id | Default | Compiled | Sortlen | +-------------+---------+------+---------+----------+---------+ @@ -39,7 +46,10 @@ mysql> SHOW COLLATION; When new collation framework is enabled, `utf8_general_ci` and `utf8mb4_general_ci` are additionally supported. ```sql -mysql> SHOW COLLATION; +SHOW COLLATION; +``` + +``` +--------------------+---------+------+---------+----------+---------+ | Collation | Charset | Id | Default | Compiled | Sortlen | +--------------------+---------+------+---------+----------+---------+ @@ -58,6 +68,25 @@ mysql> SHOW COLLATION; 11 rows in set (0.001 sec) ``` +To filter on the character set, you can add a `WHERE` clause. + +```sql +SHOW COLLATION WHERE Charset="utf8mb4"; +``` + +```sql ++--------------------+---------+-----+---------+----------+---------+ +| Collation | Charset | Id | Default | Compiled | Sortlen | ++--------------------+---------+-----+---------+----------+---------+ +| utf8mb4_0900_ai_ci | utf8mb4 | 255 | | Yes | 1 | +| utf8mb4_0900_bin | utf8mb4 | 309 | | Yes | 1 | +| utf8mb4_bin | utf8mb4 | 46 | Yes | Yes | 1 | +| utf8mb4_general_ci | utf8mb4 | 45 | | Yes | 1 | +| utf8mb4_unicode_ci | utf8mb4 | 224 | | Yes | 1 | ++--------------------+---------+-----+---------+----------+---------+ +5 rows in set (0.00 sec) +``` + ## MySQL compatibility The usage of the `SHOW COLLATION` statement in TiDB is fully compatible with MySQL. However, charsets in TiDB might have different default collations compared with MySQL. For details, refer to [Compatibility with MySQL](/mysql-compatibility.md). If you find any compatibility differences, [report a bug](https://docs.pingcap.com/tidb/stable/support). diff --git a/sql-statements/sql-statement-show-columns-from.md b/sql-statements/sql-statement-show-columns-from.md index eef35f39bf7db..1c2e3d8e29a3d 100644 --- a/sql-statements/sql-statement-show-columns-from.md +++ b/sql-statements/sql-statement-show-columns-from.md @@ -1,7 +1,6 @@ --- title: SHOW [FULL] COLUMNS FROM | TiDB SQL Statement Reference summary: An overview of the usage of SHOW [FULL] COLUMNS FROM for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-columns-from/','/docs/dev/reference/sql/statements/show-columns-from/'] --- # SHOW [FULL] COLUMNS FROM diff --git a/sql-statements/sql-statement-show-config.md b/sql-statements/sql-statement-show-config.md index bca81011141d4..ee301ec69c60c 100644 --- a/sql-statements/sql-statement-show-config.md +++ b/sql-statements/sql-statement-show-config.md @@ -1,7 +1,6 @@ --- title: SHOW CONFIG summary: Overview of the use of SHOW CONFIG in the TiDB database -aliases: ['/docs/dev/sql-statements/sql-statement-show-config/'] --- # SHOW CONFIG @@ -10,24 +9,23 @@ The `SHOW CONFIG` statement is used to show the current configuration of various > **Note:** > -> This feature is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This feature is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). ## Synopsis -**ShowStmt:** +```ebnf+diagram +ShowConfigStmt ::= + "SHOW" "CONFIG" ShowLikeOrWhere? -![ShowStmt](/media/sqlgram/ShowStmt.png) - -**ShowTargetFilterable:** - -![ShowTargetFilterable](/media/sqlgram/ShowTargetFilterable.png) +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` ## Examples Show all configurations: -{{< copyable "sql" >}} - ```sql SHOW CONFIG; ``` @@ -45,8 +43,6 @@ SHOW CONFIG; Show the configuration where the `type` is `tidb`: -{{< copyable "sql" >}} - ```sql SHOW CONFIG WHERE type = 'tidb' AND name = 'advertise-address'; ``` @@ -62,8 +58,6 @@ SHOW CONFIG WHERE type = 'tidb' AND name = 'advertise-address'; You can also use the `LIKE` clause to show the configuration where the `type` is `tidb`: -{{< copyable "sql" >}} - ```sql SHOW CONFIG LIKE 'tidb'; ``` diff --git a/sql-statements/sql-statement-show-create-database.md b/sql-statements/sql-statement-show-create-database.md index 2231295f8393a..f8202c1f0a235 100644 --- a/sql-statements/sql-statement-show-create-database.md +++ b/sql-statements/sql-statement-show-create-database.md @@ -13,7 +13,7 @@ summary: An overview of the use of SHOW CREATE DATABASE in the TiDB database. ```ebnf+diagram ShowCreateDatabaseStmt ::= - "SHOW" "CREATE" "DATABASE" | "SCHEMA" ("IF" "NOT" "EXISTS")? DBName + "SHOW" "CREATE" ("DATABASE" | "SCHEMA") ("IF" "NOT" "EXISTS")? DBName ``` ## Examples diff --git a/sql-statements/sql-statement-show-create-placement-policy.md b/sql-statements/sql-statement-show-create-placement-policy.md index cdf84b2f4802f..2908fe31aac22 100644 --- a/sql-statements/sql-statement-show-create-placement-policy.md +++ b/sql-statements/sql-statement-show-create-placement-policy.md @@ -9,7 +9,7 @@ summary: The usage of SHOW CREATE PLACEMENT POLICY in TiDB. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ## Synopsis @@ -28,7 +28,7 @@ PolicyName ::= ```sql CREATE PLACEMENT POLICY p1 PRIMARY_REGION="us-east-1" REGIONS="us-east-1,us-west-1" FOLLOWERS=4; CREATE TABLE t1 (a INT) PLACEMENT POLICY=p1; -SHOW CREATE PLACEMENT POLICY p1\G; +SHOW CREATE PLACEMENT POLICY p1\G ``` ``` diff --git a/sql-statements/sql-statement-show-create-resource-group.md b/sql-statements/sql-statement-show-create-resource-group.md index afd9a78ea2e6e..68e0226a1058b 100644 --- a/sql-statements/sql-statement-show-create-resource-group.md +++ b/sql-statements/sql-statement-show-create-resource-group.md @@ -9,7 +9,7 @@ You can use the `SHOW CREATE RESOURCE GROUP` statement to view the current defin > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-show-create-sequence.md b/sql-statements/sql-statement-show-create-sequence.md index b8b9b5f69007d..74792e4edecd1 100644 --- a/sql-statements/sql-statement-show-create-sequence.md +++ b/sql-statements/sql-statement-show-create-sequence.md @@ -1,7 +1,6 @@ --- title: SHOW CREATE SEQUENCE summary: An overview of the usage of SHOW CREATE SEQUENCE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-create-sequence/','/docs/dev/reference/sql/statements/show-create-sequence/'] --- # SHOW CREATE SEQUENCE @@ -52,4 +51,5 @@ This statement is a TiDB extension. The implementation is modeled on sequences a ## See also * [CREATE SEQUENCE](/sql-statements/sql-statement-create-sequence.md) +* [ALTER SEQUENCE](/sql-statements/sql-statement-alter-sequence.md) * [DROP SEQUENCE](/sql-statements/sql-statement-drop-sequence.md) diff --git a/sql-statements/sql-statement-show-create-table.md b/sql-statements/sql-statement-show-create-table.md index b4d58e4c74425..eb1430d105138 100644 --- a/sql-statements/sql-statement-show-create-table.md +++ b/sql-statements/sql-statement-show-create-table.md @@ -1,7 +1,6 @@ --- title: SHOW CREATE TABLE | TiDB SQL Statement Reference summary: An overview of the usage of SHOW CREATE TABLE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-create-table/','/docs/dev/reference/sql/statements/show-create-table/'] --- # SHOW CREATE TABLE diff --git a/sql-statements/sql-statement-show-create-user.md b/sql-statements/sql-statement-show-create-user.md index d43ffbce8cad7..de3535fedeb5c 100644 --- a/sql-statements/sql-statement-show-create-user.md +++ b/sql-statements/sql-statement-show-create-user.md @@ -1,7 +1,6 @@ --- title: SHOW CREATE USER | TiDB SQL Statement Reference summary: An overview of the usage of SHOW CREATE USER for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-create-user/','/docs/dev/reference/sql/statements/show-create-user/'] --- # SHOW CREATE USER diff --git a/sql-statements/sql-statement-show-databases.md b/sql-statements/sql-statement-show-databases.md index 7b6a762582bd9..74a58ded4eb93 100644 --- a/sql-statements/sql-statement-show-databases.md +++ b/sql-statements/sql-statement-show-databases.md @@ -1,7 +1,6 @@ --- title: SHOW DATABASES | TiDB SQL Statement Reference summary: An overview of the usage of SHOW DATABASES for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-databases/','/docs/dev/reference/sql/statements/show-databases/'] --- # SHOW DATABASES @@ -12,13 +11,14 @@ This statement shows a list of databases that the current user has privileges to ## Synopsis -**ShowDatabasesStmt:** +```ebnf+diagram +ShowDatabasesStmt ::= + "SHOW" "DATABASES" ShowLikeOrWhere? -![ShowDatabasesStmt](/media/sqlgram/ShowDatabasesStmt.png) - -**ShowLikeOrWhereOpt:** - -![ShowLikeOrWhereOpt](/media/sqlgram/ShowLikeOrWhereOpt.png) +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` ## Examples @@ -59,3 +59,4 @@ The `SHOW DATABASES` statement in TiDB is fully compatible with MySQL. If you fi * [SHOW SCHEMAS](/sql-statements/sql-statement-show-schemas.md) * [DROP DATABASE](/sql-statements/sql-statement-drop-database.md) * [CREATE DATABASE](/sql-statements/sql-statement-create-database.md) +* [`INFORMATION_SCHEMA.SCHEMATA`](/information-schema/information-schema-schemata.md) diff --git a/sql-statements/sql-statement-show-drainer-status.md b/sql-statements/sql-statement-show-drainer-status.md index a0dd876e99fe3..2cb36d45c4959 100644 --- a/sql-statements/sql-statement-show-drainer-status.md +++ b/sql-statements/sql-statement-show-drainer-status.md @@ -1,7 +1,6 @@ --- title: SHOW DRAINER STATUS summary: An overview of the usage of SHOW DRAINER STATUS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-drainer-status/'] --- # SHOW DRAINER STATUS @@ -10,7 +9,7 @@ The `SHOW DRAINER STATUS` statement displays the status information for all Drai > **Note:** > -> This feature is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This feature is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). ## Examples diff --git a/sql-statements/sql-statement-show-engines.md b/sql-statements/sql-statement-show-engines.md index e0234219a503b..259857c859660 100644 --- a/sql-statements/sql-statement-show-engines.md +++ b/sql-statements/sql-statement-show-engines.md @@ -1,7 +1,6 @@ --- title: SHOW ENGINES | TiDB SQL Statement Reference summary: An overview of the usage of SHOW ENGINES for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-engines/','/docs/dev/reference/sql/statements/show-engines/'] --- # SHOW ENGINES @@ -10,12 +9,13 @@ This statement is used to list all supported storage engines. The syntax is incl ## Synopsis -**ShowEnginesStmt:** +```ebnf+diagram +ShowEnginesStmt ::= + "SHOW" "ENGINES" ShowLikeOrWhere? -![ShowEnginesStmt](/media/sqlgram/ShowEnginesStmt.png) - -```sql -SHOW ENGINES; +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression ``` ## Examples @@ -32,4 +32,4 @@ mysql> SHOW ENGINES; ## MySQL compatibility -* This statement will always only return InnoDB as the supported engine. Internally, TiDB will typically use TiKV as the storage engine. +* This statement will always only return InnoDB as the supported engine. Internally, TiDB will typically use [TiKV](/tikv-overview.md) as the storage engine. diff --git a/sql-statements/sql-statement-show-errors.md b/sql-statements/sql-statement-show-errors.md index 734525262a935..0d634fe651a29 100644 --- a/sql-statements/sql-statement-show-errors.md +++ b/sql-statements/sql-statement-show-errors.md @@ -1,7 +1,6 @@ --- title: SHOW ERRORS | TiDB SQL Statement Reference summary: An overview of the usage of SHOW ERRORS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-errors/','/docs/dev/reference/sql/statements/show-errors/'] --- # SHOW ERRORS @@ -12,9 +11,14 @@ The behavior of which statements generate errors vs. warnings is highly influenc ## Synopsis -**ShowErrorsStmt:** +```ebnf+diagram +ShowErrorsStmt ::= + "SHOW" "ERRORS" ShowLikeOrWhere? -![ShowErrorsStmt](/media/sqlgram/ShowErrorsStmt.png) +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` ## Examples diff --git a/sql-statements/sql-statement-show-fields-from.md b/sql-statements/sql-statement-show-fields-from.md index 657e8bdf1aa89..85527ed6b3f90 100644 --- a/sql-statements/sql-statement-show-fields-from.md +++ b/sql-statements/sql-statement-show-fields-from.md @@ -1,7 +1,6 @@ --- title: SHOW [FULL] FIELDS FROM | TiDB SQL Statement Reference summary: An overview of the usage of SHOW [FULL] FIELDS FROM for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-fields-from/','/docs/dev/reference/sql/statements/show-fields-from/'] --- # SHOW [FULL] FIELDS FROM diff --git a/sql-statements/sql-statement-show-grants.md b/sql-statements/sql-statement-show-grants.md index 2fc346f6b9118..fe2866ab77c40 100644 --- a/sql-statements/sql-statement-show-grants.md +++ b/sql-statements/sql-statement-show-grants.md @@ -1,7 +1,6 @@ --- title: SHOW GRANTS | TiDB SQL Statement Reference summary: An overview of the usage of SHOW GRANTS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-grants/','/docs/dev/reference/sql/statements/show-grants/'] --- # SHOW GRANTS diff --git a/sql-statements/sql-statement-show-histograms.md b/sql-statements/sql-statement-show-histograms.md index 333a791532c65..091aa0e7fa677 100644 --- a/sql-statements/sql-statement-show-histograms.md +++ b/sql-statements/sql-statement-show-histograms.md @@ -1,7 +1,6 @@ --- title: SHOW STATS_HISTOGRAMS summary: An overview of the usage of SHOW HISTOGRAMS for TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-histograms/'] --- # SHOW STATS_HISTOGRAMS diff --git a/sql-statements/sql-statement-show-import-job.md b/sql-statements/sql-statement-show-import-job.md index f392edb3df1c2..396f60558a137 100644 --- a/sql-statements/sql-statement-show-import-job.md +++ b/sql-statements/sql-statement-show-import-job.md @@ -7,12 +7,6 @@ summary: An overview of the usage of SHOW IMPORT in TiDB. The `SHOW IMPORT` statement is used to show the IMPORT jobs created in TiDB. This statement can only show jobs created by the current user. - - ## Required privileges - `SHOW IMPORT JOBS`: if a user has the `SUPER` privilege, this statement shows all import jobs in TiDB. Otherwise, this statement only shows jobs created by the current user. diff --git a/sql-statements/sql-statement-show-index.md b/sql-statements/sql-statement-show-index.md deleted file mode 100644 index b4956b1a722ea..0000000000000 --- a/sql-statements/sql-statement-show-index.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: SHOW INDEX [FROM|IN] | TiDB SQL Statement Reference -summary: An overview of the usage of SHOW INDEX [FROM|IN] for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-index/','/docs/dev/reference/sql/statements/show-index/'] ---- - -# SHOW INDEX [FROM|IN] - -This statement is an alias to [`SHOW INDEXES [FROM|IN]`](/sql-statements/sql-statement-show-indexes.md). It is included for compatibility with MySQL. diff --git a/sql-statements/sql-statement-show-indexes.md b/sql-statements/sql-statement-show-indexes.md index 7a71afe37a572..e4abb46f17080 100644 --- a/sql-statements/sql-statement-show-indexes.md +++ b/sql-statements/sql-statement-show-indexes.md @@ -1,7 +1,7 @@ --- title: SHOW INDEXES [FROM|IN] | TiDB SQL Statement Reference summary: An overview of the usage of SHOW INDEXES [FROM|IN] for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-indexes/','/docs/dev/reference/sql/statements/show-indexes/'] +aliases: ['/tidb/v7.5/sql-statement-show-index/', '/tidb/v7.5/sql-statement-show-keys/'] --- # SHOW INDEXES [FROM|IN] @@ -40,10 +40,8 @@ mysql> SHOW INDEXES FROM t1; +-------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+---------+------------+ | Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | +-------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+---------+------------+ -| t1 | 0 | PRIMARY | 1 | id | A | 0 | NULL | NULL | | BTREE | | | YES - | NULL | -| t1 | 1 | col1 | 1 | col1 | A | 0 | NULL | NULL | YES | BTREE | | | YES - | NULL | +| t1 | 0 | PRIMARY | 1 | id | A | 0 | NULL | NULL | | BTREE | | | YES | NULL | +| t1 | 1 | col1 | 1 | col1 | A | 0 | NULL | NULL | YES | BTREE | | | YES | NULL | +-------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+---------+------------+ 2 rows in set (0.00 sec) @@ -51,10 +49,8 @@ mysql> SHOW INDEX FROM t1; +-------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+---------+------------+ | Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | +-------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+---------+------------+ -| t1 | 0 | PRIMARY | 1 | id | A | 0 | NULL | NULL | | BTREE | | | YES - | NULL | -| t1 | 1 | col1 | 1 | col1 | A | 0 | NULL | NULL | YES | BTREE | | | YES - | NULL | +| t1 | 0 | PRIMARY | 1 | id | A | 0 | NULL | NULL | | BTREE | | | YES | NULL | +| t1 | 1 | col1 | 1 | col1 | A | 0 | NULL | NULL | YES | BTREE | | | YES | NULL | +-------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+---------+------------+ 2 rows in set (0.00 sec) @@ -62,10 +58,8 @@ mysql> SHOW KEYS FROM t1; +-------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+---------+------------+ | Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | +-------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+---------+------------+ -| t1 | 0 | PRIMARY | 1 | id | A | 0 | NULL | NULL | | BTREE | | | YES - | NULL | -| t1 | 1 | col1 | 1 | col1 | A | 0 | NULL | NULL | YES | BTREE | | | YES - | NULL | +| t1 | 0 | PRIMARY | 1 | id | A | 0 | NULL | NULL | | BTREE | | | YES | NULL | +| t1 | 1 | col1 | 1 | col1 | A | 0 | NULL | NULL | YES | BTREE | | | YES | NULL | +-------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+---------+------------+ 2 rows in set (0.00 sec) ``` diff --git a/sql-statements/sql-statement-show-keys.md b/sql-statements/sql-statement-show-keys.md deleted file mode 100644 index 6a6bf856acffe..0000000000000 --- a/sql-statements/sql-statement-show-keys.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: SHOW KEYS [FROM|IN] | TiDB SQL Statement Reference -summary: An overview of the usage of SHOW KEYS [FROM|IN] for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-keys/','/docs/dev/reference/sql/statements/show-keys/'] ---- - -# SHOW KEYS [FROM|IN] - -This statement is an alias to [`SHOW INDEXES [FROM|IN]`](/sql-statements/sql-statement-show-indexes.md). It is included for compatibility with MySQL. diff --git a/sql-statements/sql-statement-show-master-status.md b/sql-statements/sql-statement-show-master-status.md index deb31e34da5fc..bff18ca684b83 100644 --- a/sql-statements/sql-statement-show-master-status.md +++ b/sql-statements/sql-statement-show-master-status.md @@ -1,7 +1,6 @@ --- title: SHOW MASTER STATUS summary: An overview of the usage of SHOW MASTER STATUS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-master-status/'] --- # SHOW MASTER STATUS diff --git a/sql-statements/sql-statement-show-placement-for.md b/sql-statements/sql-statement-show-placement-for.md index 31b7cd1f94665..8e130bfc6192e 100644 --- a/sql-statements/sql-statement-show-placement-for.md +++ b/sql-statements/sql-statement-show-placement-for.md @@ -9,7 +9,7 @@ summary: The usage of SHOW PLACEMENT FOR in TiDB. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. The statement returns a result set in which the `Scheduling_State` field indicates the current progress that the Placement Driver (PD) has made in scheduling the placement: @@ -39,9 +39,9 @@ ALTER DATABASE test PLACEMENT POLICY=p1; CREATE TABLE t1 (a INT); SHOW PLACEMENT FOR DATABASE test; SHOW PLACEMENT FOR TABLE t1; -SHOW CREATE TABLE t1\G; +SHOW CREATE TABLE t1\G CREATE TABLE t3 (a INT) PARTITION BY RANGE (a) (PARTITION p1 VALUES LESS THAN (10), PARTITION p2 VALUES LESS THAN (20)); -SHOW PLACEMENT FOR TABLE t3 PARTITION p1\G; +SHOW PLACEMENT FOR TABLE t3 PARTITION p1\G ``` ``` diff --git a/sql-statements/sql-statement-show-placement-labels.md b/sql-statements/sql-statement-show-placement-labels.md index 4e7c81e363a91..07bf367b17eca 100644 --- a/sql-statements/sql-statement-show-placement-labels.md +++ b/sql-statements/sql-statement-show-placement-labels.md @@ -9,7 +9,7 @@ summary: The usage of SHOW PLACEMENT LABELS in TiDB. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-show-placement.md b/sql-statements/sql-statement-show-placement.md index de27500e62869..9625e682b0f07 100644 --- a/sql-statements/sql-statement-show-placement.md +++ b/sql-statements/sql-statement-show-placement.md @@ -9,7 +9,7 @@ summary: The usage of SHOW PLACEMENT in TiDB. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. The statement returns a result set in which the `Scheduling_State` field indicates the current progress that the Placement Driver (PD) has made in scheduling the placement: diff --git a/sql-statements/sql-statement-show-plugins.md b/sql-statements/sql-statement-show-plugins.md index 660b9d83cd02e..ec8790fa7074c 100644 --- a/sql-statements/sql-statement-show-plugins.md +++ b/sql-statements/sql-statement-show-plugins.md @@ -1,7 +1,6 @@ --- title: SHOW PLUGINS summary: An overview of the usage of SHOW PLUGINS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-plugins/'] --- # SHOW PLUGINS @@ -10,22 +9,17 @@ aliases: ['/docs/dev/sql-statements/sql-statement-show-plugins/'] > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ## Synopsis -**ShowStmt:** - -![ShowStmt](/media/sqlgram/ShowStmt.png) - -**ShowTargetFilterable:** - -![ShowTargetFilterable](/media/sqlgram/ShowTargetFilterable.png) +```ebnf+diagram +ShowPluginsStmt ::= + "SHOW" "PLUGINS" ShowLikeOrWhere? +``` ## Examples -{{< copyable "sql" >}} - ```sql SHOW PLUGINS; ``` @@ -39,8 +33,6 @@ SHOW PLUGINS; 1 row in set (0.000 sec) ``` -{{< copyable "sql" >}} - ```sql SHOW PLUGINS LIKE 'a%'; ``` @@ -57,3 +49,7 @@ SHOW PLUGINS LIKE 'a%'; ## MySQL compatibility The `SHOW PLUGINS` statement in TiDB is fully compatible with MySQL. If you find any compatibility differences, [report a bug](https://docs.pingcap.com/tidb/stable/support). + +## See also + +- [`ADMIN PLUGINS`](/sql-statements/sql-statement-admin.md#admin-plugins-related-statement) \ No newline at end of file diff --git a/sql-statements/sql-statement-show-privileges.md b/sql-statements/sql-statement-show-privileges.md index 65ac4722ab61e..7c024f2194384 100644 --- a/sql-statements/sql-statement-show-privileges.md +++ b/sql-statements/sql-statement-show-privileges.md @@ -1,7 +1,6 @@ --- title: SHOW PRIVILEGES | TiDB SQL Statement Reference summary: An overview of the usage of SHOW PRIVILEGES for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-privileges/','/docs/dev/reference/sql/statements/show-privileges/'] --- # SHOW PRIVILEGES @@ -10,14 +9,18 @@ This statement shows a list of assignable privileges in TiDB. It is a static lis ## Synopsis -**ShowStmt:** - -![ShowStmt](/media/sqlgram/ShowStmt.png) +```ebnf+diagram +ShowPrivilegesStmt ::= + "SHOW" "PRIVILEGES" +``` ## Examples ```sql -mysql> show privileges; +SHOW PRIVILEGES; +``` + +``` +---------------------------------+---------------------------------------+-------------------------------------------------------+ | Privilege | Context | Comment | +---------------------------------+---------------------------------------+-------------------------------------------------------+ @@ -80,5 +83,18 @@ The `SHOW PRIVILEGES` statement in TiDB is fully compatible with MySQL. If you f ## See also + + * [SHOW GRANTS](/sql-statements/sql-statement-show-grants.md) +* [Privilege Management](/privilege-management.md) * [`GRANT `](/sql-statements/sql-statement-grant-privileges.md) + + + + + +* [SHOW GRANTS](/sql-statements/sql-statement-show-grants.md) +* [Privilege Management](https://docs.pingcap.com/tidb/stable/privilege-management) +* [`GRANT `](/sql-statements/sql-statement-grant-privileges.md) + + diff --git a/sql-statements/sql-statement-show-processlist.md b/sql-statements/sql-statement-show-processlist.md index 2614426960139..e6f806d6d2ce0 100644 --- a/sql-statements/sql-statement-show-processlist.md +++ b/sql-statements/sql-statement-show-processlist.md @@ -1,7 +1,6 @@ --- title: SHOW [FULL] PROCESSLIST | TiDB SQL Statement Reference summary: An overview of the usage of SHOW [FULL] PROCESSLIST for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-processlist/','/docs/dev/reference/sql/statements/show-processlist/'] --- # SHOW [FULL] PROCESSLIST @@ -10,13 +9,10 @@ This statement lists the current sessions connected to the same TiDB server. The ## Synopsis -**ShowProcesslistStmt:** - -![ShowProcesslistStmt](/media/sqlgram/ShowProcesslistStmt.png) - -**OptFull:** - -![OptFull](/media/sqlgram/OptFull.png) +```ebnf+diagram +ShowProcesslistStmt ::= + "SHOW" "FULL"? "PROCESSLIST" +``` ## Examples @@ -37,3 +33,4 @@ mysql> SHOW PROCESSLIST; ## See also * [KILL \[TIDB\]](/sql-statements/sql-statement-kill.md) +* [`INFORMATION_SCHEMA.PROCESSLIST`](/information-schema/information-schema-processlist.md) diff --git a/sql-statements/sql-statement-show-profiles.md b/sql-statements/sql-statement-show-profiles.md index fe5c67a7d5bc1..a1f64594483f3 100644 --- a/sql-statements/sql-statement-show-profiles.md +++ b/sql-statements/sql-statement-show-profiles.md @@ -1,7 +1,6 @@ --- title: SHOW PROFILES summary: An overview of the usage of SHOW PROFILES for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-profiles/'] --- # SHOW PROFILES diff --git a/sql-statements/sql-statement-show-pump-status.md b/sql-statements/sql-statement-show-pump-status.md index c232df19fb654..6c950f94d30f9 100644 --- a/sql-statements/sql-statement-show-pump-status.md +++ b/sql-statements/sql-statement-show-pump-status.md @@ -1,7 +1,6 @@ --- title: SHOW PUMP STATUS summary: An overview of the usage of SHOW PUMP STATUS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-pump-status/'] --- # SHOW PUMP STATUS @@ -10,7 +9,7 @@ The `SHOW PUMP STATUS` statement displays the status information for all Pump no > **Note:** > -> This feature is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This feature is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). ## Examples diff --git a/sql-statements/sql-statement-show-schemas.md b/sql-statements/sql-statement-show-schemas.md index 5daedb7a5f670..58ec99ad3a3d2 100644 --- a/sql-statements/sql-statement-show-schemas.md +++ b/sql-statements/sql-statement-show-schemas.md @@ -1,7 +1,6 @@ --- title: SHOW SCHEMAS | TiDB SQL Statement Reference summary: An overview of the usage of SHOW SCHEMAS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-schemas/','/docs/dev/reference/sql/statements/show-schemas/'] --- # SHOW SCHEMAS diff --git a/sql-statements/sql-statement-show-stats-locked.md b/sql-statements/sql-statement-show-stats-locked.md index 51e026bdcc8da..a3f9d66e41273 100644 --- a/sql-statements/sql-statement-show-stats-locked.md +++ b/sql-statements/sql-statement-show-stats-locked.md @@ -7,10 +7,6 @@ summary: An overview of the usage of SHOW STATS_LOCKED for the TiDB database. `SHOW STATS_LOCKED` shows the tables whose statistics are locked. -> **Warning:** -> -> Locking statistics is an experimental feature for the current version. It is not recommended to use it in the production environment. - ## Synopsis ```ebnf+diagram diff --git a/sql-statements/sql-statement-show-stats-meta.md b/sql-statements/sql-statement-show-stats-meta.md index 20f1c25202878..af102b4d3d99a 100644 --- a/sql-statements/sql-statement-show-stats-meta.md +++ b/sql-statements/sql-statement-show-stats-meta.md @@ -1,7 +1,6 @@ --- title: SHOW STATS_META summary: An overview of the usage of SHOW STATS_META for TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-stats-meta/'] --- # SHOW STATS_META @@ -25,24 +24,19 @@ Currently, the `SHOW STATS_META` statement outputs 6 columns: ## Synopsis -**ShowStmt** +```ebnf+diagram +ShowStatsMetaStmt ::= + "SHOW" "STATS_META" ShowLikeOrWhere? -![ShowStmt](/media/sqlgram/ShowStmt.png) - -**ShowTargetFiltertable** - -![ShowTargetFilterable](/media/sqlgram/ShowTargetFilterable.png) - -**ShowLikeOrWhereOpt** - -![ShowLikeOrWhereOpt](/media/sqlgram/ShowLikeOrWhereOpt.png) +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` ## Examples -{{< copyable "sql" >}} - ```sql -show stats_meta; +SHOW STATS_META; ``` ```sql @@ -58,10 +52,8 @@ show stats_meta; 5 rows in set (0.00 sec) ``` -{{< copyable "sql" >}} - ```sql -show stats_meta where table_name = 't2'; +SHOW STATS_META WHERE table_name = 't2'; ``` ```sql diff --git a/sql-statements/sql-statement-show-status.md b/sql-statements/sql-statement-show-status.md index 21ea4c7dc538e..9d88a21dd837c 100644 --- a/sql-statements/sql-statement-show-status.md +++ b/sql-statements/sql-statement-show-status.md @@ -1,61 +1,70 @@ --- title: SHOW [GLOBAL|SESSION] STATUS | TiDB SQL Statement Reference summary: An overview of the usage of SHOW [GLOBAL|SESSION] STATUS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-status/','/docs/dev/reference/sql/statements/show-status/'] --- # SHOW [GLOBAL|SESSION] STATUS -This statement is included for compatibility with MySQL. It has no effect on TiDB, which uses Prometheus and Grafana for centralized metrics collection instead of `SHOW STATUS`. +This statement is included for compatibility with MySQL. TiDB uses Prometheus and Grafana for centralized metrics collection instead of `SHOW STATUS` for most metrics. -## Synopsis - -**ShowStmt:** - -![ShowStmt](/media/sqlgram/ShowStmt.png) +A full description of the variables can be found here: [status variables](/status-variables.md) -**ShowTargetFilterable:** +## Synopsis -![ShowTargetFilterable](/media/sqlgram/ShowTargetFilterable.png) +```ebnf+diagram +ShowStatusStmt ::= + 'SHOW' Scope? 'STATUS' ShowLikeOrWhere? -**GlobalScope:** +Scope ::= + ( 'GLOBAL' | 'SESSION' ) -![GlobalScope](/media/sqlgram/GlobalScope.png) +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` ## Examples ```sql -mysql> show status; -+--------------------+--------------------------------------+ -| Variable_name | Value | -+--------------------+--------------------------------------+ -| Ssl_cipher_list | | -| server_id | 93e2e07d-6bb4-4a1b-90b7-e035fae154fe | -| ddl_schema_version | 141 | -| Ssl_verify_mode | 0 | -| Ssl_version | | -| Ssl_cipher | | -+--------------------+--------------------------------------+ -6 rows in set (0.01 sec) - -mysql> show global status; -+--------------------+--------------------------------------+ -| Variable_name | Value | -+--------------------+--------------------------------------+ -| Ssl_cipher | | -| Ssl_cipher_list | | -| Ssl_verify_mode | 0 | -| Ssl_version | | -| server_id | 93e2e07d-6bb4-4a1b-90b7-e035fae154fe | -| ddl_schema_version | 141 | -+--------------------+--------------------------------------+ -6 rows in set (0.00 sec) +mysql> SHOW SESSION STATUS; ++-------------------------------+--------------------------------------+ +| Variable_name | Value | ++-------------------------------+--------------------------------------+ +| Ssl_cipher | | +| Ssl_cipher_list | | +| Ssl_server_not_after | | +| Ssl_server_not_before | | +| Ssl_verify_mode | 0 | +| Ssl_version | | +| Uptime | 1409 | +| ddl_schema_version | 116 | +| last_plan_binding_update_time | 0000-00-00 00:00:00 | +| server_id | 61160e73-ab80-40ff-8f33-27d55d475fd1 | ++-------------------------------+--------------------------------------+ +10 rows in set (0.00 sec) + +mysql> SHOW GLOBAL STATUS; ++-----------------------+--------------------------------------+ +| Variable_name | Value | ++-----------------------+--------------------------------------+ +| Ssl_cipher | | +| Ssl_cipher_list | | +| Ssl_server_not_after | | +| Ssl_server_not_before | | +| Ssl_verify_mode | 0 | +| Ssl_version | | +| Uptime | 1413 | +| ddl_schema_version | 116 | +| server_id | 61160e73-ab80-40ff-8f33-27d55d475fd1 | ++-----------------------+--------------------------------------+ +9 rows in set (0.00 sec) ``` ## MySQL compatibility -* This statement is included only for compatibility with MySQL. +* This statement is compatible with MySQL. ## See also * [FLUSH STATUS](/sql-statements/sql-statement-flush-status.md) +* [Server Status Variables](/status-variables.md) diff --git a/sql-statements/sql-statement-show-table-next-rowid.md b/sql-statements/sql-statement-show-table-next-rowid.md index 98171492513f7..0d4845cc246a8 100644 --- a/sql-statements/sql-statement-show-table-next-rowid.md +++ b/sql-statements/sql-statement-show-table-next-rowid.md @@ -1,14 +1,13 @@ --- title: SHOW TABLE NEXT_ROW_ID summary: Learn the usage of `SHOW TABLE NEXT_ROW_ID` in TiDB. -aliases: ['/docs/dev/sql-statements/sql-statement-show-table-next-rowid/'] --- # SHOW TABLE NEXT_ROW_ID `SHOW TABLE NEXT_ROW_ID` is used to show the details of some special columns of a table, including: -* `AUTO_INCREMENT` column automatically created by TiDB, namely, `_tidb_rowid` column. +* [`_tidb_rowid`](https://docs.pingcap.com/tidb/v7.5/tidb-rowid/), the hidden row ID column automatically managed by TiDB for supported tables. * `AUTO_INCREMENT` column created by users. * [`AUTO_RANDOM`](/auto-random.md) column created by users. * [`SEQUENCE`](/sql-statements/sql-statement-create-sequence.md) created by users. @@ -71,3 +70,4 @@ This statement is a TiDB extension to MySQL syntax. * [CREATE TABLE](/sql-statements/sql-statement-create-table.md) * [AUTO_RANDOM](/auto-random.md) * [CREATE_SEQUENCE](/sql-statements/sql-statement-create-sequence.md) +* [_tidb_rowid](https://docs.pingcap.com/tidb/v7.5/tidb-rowid/) diff --git a/sql-statements/sql-statement-show-table-regions.md b/sql-statements/sql-statement-show-table-regions.md index 7270900787702..0ac4513cc0f71 100644 --- a/sql-statements/sql-statement-show-table-regions.md +++ b/sql-statements/sql-statement-show-table-regions.md @@ -1,7 +1,6 @@ --- title: SHOW TABLE REGIONS summary: Learn how to use SHOW TABLE REGIONS in TiDB. -aliases: ['/docs/dev/sql-statements/sql-statement-show-table-regions/','/docs/dev/reference/sql/statements/show-table-regions/'] --- # SHOW TABLE REGIONS @@ -10,7 +9,7 @@ The `SHOW TABLE REGIONS` statement is used to show the Region information of a t > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ## Syntax diff --git a/sql-statements/sql-statement-show-table-status.md b/sql-statements/sql-statement-show-table-status.md index 8ed264627624b..6c88eaa379783 100644 --- a/sql-statements/sql-statement-show-table-status.md +++ b/sql-statements/sql-statement-show-table-status.md @@ -1,7 +1,6 @@ --- title: SHOW TABLE STATUS | TiDB SQL Statement Reference summary: An overview of the usage of SHOW TABLE STATUS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-table-status/','/docs/dev/reference/sql/statements/show-table-status/'] --- # SHOW TABLE STATUS diff --git a/sql-statements/sql-statement-show-tables.md b/sql-statements/sql-statement-show-tables.md index 0b01ca64d2f85..3f5f9c4c9a171 100644 --- a/sql-statements/sql-statement-show-tables.md +++ b/sql-statements/sql-statement-show-tables.md @@ -1,7 +1,6 @@ --- title: SHOW [FULL] TABLES | TiDB SQL Statement Reference summary: An overview of the usage of SHOW [FULL] TABLES for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-tables/','/docs/dev/reference/sql/statements/show-tables/'] --- # SHOW [FULL] TABLES diff --git a/sql-statements/sql-statement-show-variables.md b/sql-statements/sql-statement-show-variables.md index 04530f2edc9e3..400f07d227979 100644 --- a/sql-statements/sql-statement-show-variables.md +++ b/sql-statements/sql-statement-show-variables.md @@ -1,7 +1,6 @@ --- title: SHOW [GLOBAL|SESSION] VARIABLES | TiDB SQL Statement Reference summary: An overview of the usage of SHOW [GLOBAL|SESSION] VARIABLES for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-variables/','/docs/dev/reference/sql/statements/show-variables/'] --- # SHOW [GLOBAL|SESSION] VARIABLES @@ -10,134 +9,36 @@ This statement shows a list of variables for the scope of either `GLOBAL` or `SE ## Synopsis -**ShowStmt:** - -![ShowStmt](/media/sqlgram/ShowStmt.png) - -**ShowTargetFilterable:** - -![ShowTargetFilterable](/media/sqlgram/ShowTargetFilterable.png) - -**GlobalScope:** - -![GlobalScope](/media/sqlgram/GlobalScope.png) +```ebnf+diagram +ShowVariablesStmt ::= + "SHOW" ("GLOBAL" | "SESSION")? VARIABLES ShowLikeOrWhere? + +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` ## Examples -List all TiDB specific variables. For detailed description, refer to [System Variables](/system-variables.md). +The following example demonstrates how to use the `SHOW [GLOBAL|SESSION] VARIABLES` statement to show variables whose names or values match a specific pattern. For detailed descriptions of these variables, see [System Variables](/system-variables.md). ```sql -mysql> SHOW GLOBAL VARIABLES LIKE 'tidb%'; +mysql> SHOW GLOBAL VARIABLES LIKE 'tidb_stmt_summary%'; +-------------------------------------+---------------------+ | Variable_name | Value | +-------------------------------------+---------------------+ -| tidb_allow_batch_cop | 0 | -| tidb_allow_remove_auto_inc | 0 | -| tidb_auto_analyze_end_time | 23:59 +0000 | -| tidb_auto_analyze_ratio | 0.5 | -| tidb_auto_analyze_start_time | 00:00 +0000 | -| tidb_backoff_lock_fast | 100 | -| tidb_backoff_weight | 2 | -| tidb_batch_commit | 0 | -| tidb_batch_delete | 0 | -| tidb_build_stats_concurrency | 4 | -| tidb_capture_plan_baselines | off | -| tidb_check_mb4_value_in_utf8 | 1 | -| tidb_checksum_table_concurrency | 4 | -| tidb_config | | -| tidb_constraint_check_in_place | 0 | -| tidb_current_ts | 0 | -| tidb_ddl_error_count_limit | 512 | -| tidb_ddl_reorg_batch_size | 256 | -| tidb_ddl_reorg_priority | PRIORITY_LOW | -| tidb_ddl_reorg_worker_cnt | 4 | -| tidb_disable_txn_auto_retry | 1 | -| tidb_distsql_scan_concurrency | 15 | -| tidb_dml_batch_size | 20000 | -| tidb_enable_cascades_planner | 0 | -| tidb_enable_chunk_rpc | 1 | -| tidb_enable_collect_execution_info | 1 | -| tidb_enable_index_merge | 0 | -| tidb_enable_noop_functions | 0 | -| tidb_enable_radix_join | 0 | -| tidb_enable_slow_log | 1 | -| tidb_enable_stmt_summary | 1 | -| tidb_enable_table_partition | on | -| tidb_enable_vectorized_expression | 1 | -| tidb_enable_window_function | 1 | -| tidb_evolve_plan_baselines | off | -| tidb_evolve_plan_task_end_time | 23:59 +0000 | -| tidb_evolve_plan_task_max_time | 600 | -| tidb_evolve_plan_task_start_time | 00:00 +0000 | -| tidb_expensive_query_time_threshold | 60 | -| tidb_force_priority | NO_PRIORITY | -| tidb_general_log | 0 | -| tidb_hash_join_concurrency | 5 | -| tidb_hashagg_final_concurrency | 4 | -| tidb_hashagg_partial_concurrency | 4 | -| tidb_index_join_batch_size | 25000 | -| tidb_index_lookup_concurrency | 4 | -| tidb_index_lookup_join_concurrency | 4 | -| tidb_index_lookup_size | 20000 | -| tidb_index_serial_scan_concurrency | 1 | -| tidb_init_chunk_size | 32 | -| tidb_isolation_read_engines | tikv, tiflash, tidb | -| tidb_low_resolution_tso | 0 | -| tidb_max_chunk_size | 1024 | -| tidb_max_delta_schema_count | 1024 | -| tidb_mem_quota_hashjoin | 34359738368 | -| tidb_mem_quota_indexlookupjoin | 34359738368 | -| tidb_mem_quota_indexlookupreader | 34359738368 | -| tidb_mem_quota_mergejoin | 34359738368 | -| tidb_mem_quota_nestedloopapply | 34359738368 | -| tidb_mem_quota_query | 1073741824 | -| tidb_mem_quota_sort | 34359738368 | -| tidb_mem_quota_topn | 34359738368 | -| tidb_metric_query_range_duration | 60 | -| tidb_metric_query_step | 60 | -| tidb_opt_agg_push_down | 0 | -| tidb_opt_concurrency_factor | 3 | -| tidb_opt_copcpu_factor | 3 | -| tidb_opt_correlation_exp_factor | 1 | -| tidb_opt_correlation_threshold | 0.9 | -| tidb_opt_cpu_factor | 3 | -| tidb_opt_desc_factor | 3 | -| tidb_opt_disk_factor | 1.5 | -| tidb_opt_distinct_agg_push_down | 0 | -| tidb_opt_insubq_to_join_and_agg | 1 | -| tidb_opt_join_reorder_threshold | 0 | -| tidb_opt_memory_factor | 0.001 | -| tidb_opt_network_factor | 1 | -| tidb_opt_scan_factor | 1.5 | -| tidb_opt_seek_factor | 20 | -| tidb_opt_write_row_id | 0 | -| tidb_optimizer_selectivity_level | 0 | -| tidb_pprof_sql_cpu | 0 | -| tidb_projection_concurrency | 4 | -| tidb_query_log_max_len | 4096 | -| tidb_record_plan_in_slow_log | 1 | -| tidb_replica_read | leader | -| tidb_retry_limit | 10 | -| tidb_row_format_version | 2 | -| tidb_scatter_region | 0 | -| tidb_skip_isolation_level_check | 0 | -| tidb_skip_utf8_check | 0 | -| tidb_slow_log_threshold | 300 | -| tidb_slow_query_file | tidb-slow.log | -| tidb_snapshot | | +| tidb_stmt_summary_enable_persistent | OFF | +| tidb_stmt_summary_file_max_backups | 0 | +| tidb_stmt_summary_file_max_days | 3 | +| tidb_stmt_summary_file_max_size | 64 | +| tidb_stmt_summary_filename | tidb-statements.log | | tidb_stmt_summary_history_size | 24 | -| tidb_stmt_summary_internal_query | 0 | +| tidb_stmt_summary_internal_query | OFF | | tidb_stmt_summary_max_sql_length | 4096 | | tidb_stmt_summary_max_stmt_count | 3000 | | tidb_stmt_summary_refresh_interval | 1800 | -| tidb_store_limit | 0 | -| tidb_txn_mode | | -| tidb_use_plan_baselines | on | -| tidb_wait_split_region_finish | 1 | -| tidb_wait_split_region_timeout | 300 | -| tidb_window_concurrency | 4 | +-------------------------------------+---------------------+ -108 rows in set (0.01 sec) +10 rows in set (0.001 sec) mysql> SHOW GLOBAL VARIABLES LIKE 'time_zone%'; +---------------+--------+ @@ -146,6 +47,28 @@ mysql> SHOW GLOBAL VARIABLES LIKE 'time_zone%'; | time_zone | SYSTEM | +---------------+--------+ 1 row in set (0.00 sec) + +mysql> SHOW VARIABLES WHERE Variable_name="tidb_window_concurrency"; ++-------------------------+-------+ +| Variable_name | Value | ++-------------------------+-------+ +| tidb_window_concurrency | -1 | ++-------------------------+-------+ +1 row in set (0.00 sec) + +mysql> SHOW VARIABLES WHERE Value=300; ++--------------------------------+-------+ +| Variable_name | Value | ++--------------------------------+-------+ +| ddl_slow_threshold | 300 | +| delayed_insert_timeout | 300 | +| innodb_purge_batch_size | 300 | +| key_cache_age_threshold | 300 | +| slave_checkpoint_period | 300 | +| tidb_slow_log_threshold | 300 | +| tidb_wait_split_region_timeout | 300 | ++--------------------------------+-------+ +7 rows in set (0.00 sec) ``` ## MySQL compatibility diff --git a/sql-statements/sql-statement-show-warnings.md b/sql-statements/sql-statement-show-warnings.md index dd3aa6d7aa524..a61a02401b576 100644 --- a/sql-statements/sql-statement-show-warnings.md +++ b/sql-statements/sql-statement-show-warnings.md @@ -1,7 +1,6 @@ --- title: SHOW WARNINGS | TiDB SQL Statement Reference summary: An overview of the usage of SHOW WARNINGS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-warnings/','/docs/dev/reference/sql/statements/show-warnings/'] --- # SHOW WARNINGS @@ -10,9 +9,10 @@ This statement shows a list of warnings that occurred for previously executed st ## Synopsis -**ShowWarningsStmt:** - -![ShowWarningsStmt](/media/sqlgram/ShowWarningsStmt.png) +```ebnf+diagram +ShowWarningsStmt ::= + "SHOW" "WARNINGS" +``` ## Examples diff --git a/sql-statements/sql-statement-shutdown.md b/sql-statements/sql-statement-shutdown.md index 770aaa40da948..e55143883d0ff 100644 --- a/sql-statements/sql-statement-shutdown.md +++ b/sql-statements/sql-statement-shutdown.md @@ -1,7 +1,6 @@ --- title: SHUTDOWN summary: An overview of the usage of SHUTDOWN for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-shutdown/'] --- # SHUTDOWN @@ -10,18 +9,17 @@ The `SHUTDOWN` statement is used to perform a shutdown operation in TiDB. Execut > **Note:** > -> This feature is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This feature is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). ## Synopsis -**Statement:** - -![Statement](/media/sqlgram/ShutdownStmt.png) +```ebnf+diagram +ShutdownStmt ::= + "SHUTDOWN" +``` ## Examples -{{< copyable "sql" >}} - ```sql SHUTDOWN; ``` diff --git a/sql-statements/sql-statement-split-region.md b/sql-statements/sql-statement-split-region.md index 5dda9c9cee9cc..436cc7f8cc518 100644 --- a/sql-statements/sql-statement-split-region.md +++ b/sql-statements/sql-statement-split-region.md @@ -1,7 +1,6 @@ --- title: Split Region summary: An overview of the usage of Split Region for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-split-region/','/docs/dev/reference/sql/statements/split-region/'] --- # Split Region @@ -14,7 +13,7 @@ To solve the hotspot problem in the above scenario, TiDB introduces the pre-spli > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ## Synopsis @@ -412,11 +411,11 @@ You can specify the partition to be split. ## pre_split_regions -To have evenly split Regions when a table is created, it is recommended you use `SHARD_ROW_ID_BITS` together with `PRE_SPLIT_REGIONS`. When a table is created successfully, `PRE_SPLIT_REGIONS` pre-spilts tables into the number of Regions as specified by `2^(PRE_SPLIT_REGIONS)`. +When creating a table with the `AUTO_RANDOM` or `SHARD_ROW_ID_BITS` attribute, you can also specify the `PRE_SPLIT_REGIONS` option if you want to evenly pre-split the table into Regions immediately after the table is created. The number of pre-split Regions for a table is `2^(PRE_SPLIT_REGIONS)`. > **Note:** > -> The value of `PRE_SPLIT_REGIONS` must be less than or equal to that of `SHARD_ROW_ID_BITS`. +> The value of `PRE_SPLIT_REGIONS` must be less than or equal to that of `SHARD_ROW_ID_BITS` or `AUTO_RANDOM`. The `tidb_scatter_region` global variable affects the behavior of `PRE_SPLIT_REGIONS`. This variable controls whether to wait for Regions to be pre-split and scattered before returning results after the table creation. If there are intensive writes after creating the table, you need to set the value of this variable to `1`, then TiDB will not return the results to the client until all the Regions are split and scattered. Otherwise, TiDB writes the data before the scattering is completed, which will have a significant impact on write performance. @@ -443,7 +442,7 @@ region4: [ 3<<61 , +inf ) > **Note:** > -> The Region split by the Split Region statement is controlled by the [Region merge](/best-practices/pd-scheduling-best-practices.md#region-merge) scheduler in PD. To avoid PD re-merging the newly split Region soon after, you need to [dynamically modify](/pd-control.md) configuration items related to the Region merge feature. +> The Region split by the Split Region statement is controlled by the [Region merge](/best-practices/pd-scheduling-best-practices.md#region-merge) scheduler in PD. To avoid PD re-merging the newly split Region soon after, you need to use [table attributes](/table-attributes.md) or [dynamically modify](/pd-control.md) configuration items related to the Region merge feature. diff --git a/sql-statements/sql-statement-start-transaction.md b/sql-statements/sql-statement-start-transaction.md index b943d8f2c69dd..a885dc24912b2 100644 --- a/sql-statements/sql-statement-start-transaction.md +++ b/sql-statements/sql-statement-start-transaction.md @@ -1,7 +1,6 @@ --- title: START TRANSACTION | TiDB SQL Statement Reference summary: An overview of the usage of START TRANSACTION for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-start-transaction/','/docs/dev/reference/sql/statements/start-transaction/'] --- # START TRANSACTION diff --git a/sql-statements/sql-statement-trace.md b/sql-statements/sql-statement-trace.md index 07c87bfde5afd..5fcd74e03645a 100644 --- a/sql-statements/sql-statement-trace.md +++ b/sql-statements/sql-statement-trace.md @@ -1,7 +1,6 @@ --- title: TRACE | TiDB SQL Statement Reference summary: An overview of the usage of TRACE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-trace/','/docs/dev/reference/sql/statements/trace/'] --- # TRACE @@ -10,20 +9,26 @@ The `TRACE` statement provides detailed information about query execution. It is ## Synopsis -**TraceStmt:** +```ebnf+diagram +TraceStmt ::= + "TRACE" ( "FORMAT" "=" stringLit )? TracableStmt -![TraceStmt](/media/sqlgram/TraceStmt.png) - -**TraceableStmt:** +TracableStmt ::= + ( SelectStmt | DeleteFromStmt | UpdateStmt | InsertIntoStmt | ReplaceIntoStmt | UnionStmt | LoadDataStmt | BeginTransactionStmt | CommitStmt | RollbackStmt | SetStmt ) +``` -![TraceableStmt](/media/sqlgram/TraceableStmt.png) +| Format | Description | +|--------|------------------------------------| +| row | Output in a tree format | +| json | Structured output in JSON format | +| log | Log based output | ## Examples -{{< copyable "sql" >}} +### Row ```sql -trace format='row' select * from mysql.user; +TRACE FORMAT='row' SELECT * FROM mysql.user; ``` ``` @@ -47,10 +52,10 @@ trace format='row' select * from mysql.user; 13 rows in set (0.00 sec) ``` -{{< copyable "sql" >}} +### JSON ```sql -trace format='json' select * from mysql.user; +TRACE FORMAT='json' SELECT * FROM mysql.user; ``` The JSON formatted trace can be pasted into the trace viewer, which is accessed via the TiDB status port: @@ -59,6 +64,34 @@ The JSON formatted trace can be pasted into the trace viewer, which is accessed ![TiDB Trace Viewer-2](/media/trace-view.png) +### Log + +```sql +TRACE FORMAT='log' SELECT * FROM mysql.user; +``` + +``` ++----------------------------+--------------------------------------------------------+------+------------------------------------+ +| time | event | tags | spanName | ++----------------------------+--------------------------------------------------------+------+------------------------------------+ +| 2024-04-08 08:41:47.358734 | --- start span trace ---- | | trace | +| 2024-04-08 08:41:47.358737 | --- start span session.ExecuteStmt ---- | | session.ExecuteStmt | +| 2024-04-08 08:41:47.358746 | --- start span executor.Compile ---- | | executor.Compile | +| 2024-04-08 08:41:47.358984 | --- start span session.runStmt ---- | | session.runStmt | +| 2024-04-08 08:41:47.359035 | --- start span TableReaderExecutor.Open ---- | | TableReaderExecutor.Open | +| 2024-04-08 08:41:47.359047 | --- start span distsql.Select ---- | | distsql.Select | +| 2024-04-08 08:41:47.359073 | --- start span *executor.TableReaderExecutor.Next ---- | | *executor.TableReaderExecutor.Next | +| 2024-04-08 08:41:47.359077 | table scan table: user, range: [[-inf,+inf]] | | *executor.TableReaderExecutor.Next | +| 2024-04-08 08:41:47.359094 | --- start span regionRequest.SendReqCtx ---- | | regionRequest.SendReqCtx | +| 2024-04-08 08:41:47.359098 | send Cop request to region 16 at store1 | | regionRequest.SendReqCtx | +| 2024-04-08 08:41:47.359237 | --- start span *executor.TableReaderExecutor.Next ---- | | *executor.TableReaderExecutor.Next | +| 2024-04-08 08:41:47.359240 | table scan table: user, range: [[-inf,+inf]] | | *executor.TableReaderExecutor.Next | +| 2024-04-08 08:41:47.359242 | execute done, ReturnRow: 1, ModifyRow: 0 | | trace | +| 2024-04-08 08:41:47.359252 | execute done, modify row: 0 | | trace | ++----------------------------+--------------------------------------------------------+------+------------------------------------+ +14 rows in set (0.0008 sec) +``` + ## MySQL compatibility This statement is a TiDB extension to MySQL syntax. diff --git a/sql-statements/sql-statement-truncate.md b/sql-statements/sql-statement-truncate.md index 0fa9b6490c7dd..fa8d9ab182898 100644 --- a/sql-statements/sql-statement-truncate.md +++ b/sql-statements/sql-statement-truncate.md @@ -1,7 +1,6 @@ --- title: TRUNCATE | TiDB SQL Statement Reference summary: An overview of the usage of TRUNCATE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-truncate/','/docs/dev/reference/sql/statements/truncate/'] --- # TRUNCATE diff --git a/sql-statements/sql-statement-unlock-stats.md b/sql-statements/sql-statement-unlock-stats.md index 059229a2889be..cd6446a8bafaf 100644 --- a/sql-statements/sql-statement-unlock-stats.md +++ b/sql-statements/sql-statement-unlock-stats.md @@ -7,10 +7,6 @@ summary: An overview of the usage of UNLOCK STATS for the TiDB database. `UNLOCK STATS` is used to unlock the statistics of a table or tables. -> **Warning:** -> -> Locking statistics is an experimental feature for the current version. It is not recommended to use it in the production environment. - ## Synopsis ```ebnf+diagram diff --git a/sql-statements/sql-statement-update.md b/sql-statements/sql-statement-update.md index 903de2e3230da..0e8f2d088abbf 100644 --- a/sql-statements/sql-statement-update.md +++ b/sql-statements/sql-statement-update.md @@ -1,7 +1,6 @@ --- title: UPDATE | TiDB SQL Statement Reference summary: An overview of the usage of UPDATE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-update/','/docs/dev/reference/sql/statements/update/'] --- # UPDATE @@ -34,6 +33,10 @@ The `UPDATE` statement is used to modify data in a specified table. ![WhereClauseOptional](/media/sqlgram/WhereClauseOptional.png) +> **Note:** +> +> Starting from v6.6.0, TiDB supports [Resource Control](/tidb-resource-control.md). You can use this feature to execute SQL statements with different priorities in different resource groups. By configuring proper quotas and priorities for these resource groups, you can gain better scheduling control for SQL statements with different priorities. When resource control is enabled, statement priority (`LOW_PRIORITY` and `HIGH_PRIORITY`) will no longer take effect. It is recommended that you use [Resource Control](/tidb-resource-control.md) to manage resource usage for different SQL statements. + ## Examples ```sql diff --git a/sql-statements/sql-statement-use.md b/sql-statements/sql-statement-use.md index 1cd01af142f26..868f0f0a65f7a 100644 --- a/sql-statements/sql-statement-use.md +++ b/sql-statements/sql-statement-use.md @@ -1,7 +1,6 @@ --- title: USE | TiDB SQL Statement Reference summary: An overview of the usage of USE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-use/','/docs/dev/reference/sql/statements/use/'] --- # USE diff --git a/sql-tuning-overview.md b/sql-tuning-overview.md index 9fb5bb9af847d..c329581d5db69 100644 --- a/sql-tuning-overview.md +++ b/sql-tuning-overview.md @@ -1,5 +1,6 @@ --- title: SQL Tuning Overview +summary: SQL is a declarative language, meaning it describes the final result, not the steps to execute. TiDB optimizes execution and can execute parts of the query in any order. It's similar to GPS navigation, using statistics and live traffic data. Concepts include understanding query execution plans, SQL optimization, and controlling execution plans for better performance. --- # SQL Tuning Overview diff --git a/statement-summary-tables.md b/statement-summary-tables.md index 8b28afdd2a57b..831d643cb8707 100644 --- a/statement-summary-tables.md +++ b/statement-summary-tables.md @@ -1,7 +1,6 @@ --- title: Statement Summary Tables summary: Learn about Statement Summary Table in TiDB. -aliases: ['/docs/dev/statement-summary-tables/','/docs/dev/reference/performance/statement-summary/'] --- # Statement Summary Tables @@ -18,13 +17,13 @@ Therefore, starting from v4.0.0-rc.1, TiDB provides system tables in `informatio > **Note:** > -> The preceding tables are not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> The preceding tables are not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. This document details these tables and introduces how to use them to troubleshoot SQL performance issues. ## `statements_summary` -`statements_summary` is a system table in `information_schema`. `statements_summary` groups the SQL statements by the SQL digest and the plan digest, and provides statistics for each SQL category. +`statements_summary` is a system table in `information_schema`. `statements_summary` groups the SQL statements by the resource group, the SQL digest and the plan digest, and provides statistics for each SQL category. The "SQL digest" here means the same as used in slow logs, which is a unique identifier calculated through normalized SQL statements. The normalization process ignores constant, blank characters, and is case insensitive. Therefore, statements with consistent syntaxes have the same digest. For example: @@ -86,7 +85,8 @@ The following is a sample output of querying `statements_summary`: > **Note:** > -> In TiDB, the time unit of fields in statement summary tables is nanosecond (ns), whereas in MySQL the time unit is picosecond (ps). +> - In TiDB, the time unit of fields in statement summary tables is nanosecond (ns), whereas in MySQL the time unit is picosecond (ps). +> - Starting from v7.5.1, for clusters with [resource control](/tidb-resource-control.md) enabled, `statements_summary` will be aggregated by resource group, for example, the same statements executed in different resource groups will be collected as different records. ## `statements_summary_history` @@ -209,7 +209,7 @@ To address this issue, TiDB v6.6.0 experimentally introduces the [statement summ -This section is only applicable to TiDB Self-Hosted. For TiDB Cloud, the value of the `tidb_stmt_summary_enable_persistent` parameter is `false` by default and does not support dynamic modification. +This section is only applicable to TiDB Self-Managed. For TiDB Cloud, the value of the `tidb_stmt_summary_enable_persistent` parameter is `false` by default and does not support dynamic modification. @@ -328,7 +328,7 @@ Basic fields: - `SAMPLE_USER`: The users who execute SQL statements of this category. Only one user is taken. - `PLAN_DIGEST`: The digest of the execution plan. - `PLAN`: The original execution plan. If there are multiple statements, the plan of only one statement is taken. -- `BINARY_PLAN`: The original execution plan encoded in binary format. If there are multiple statements, the plan of only one statement is taken. Execute the `SELECT tidb_decode_binary_plan('xxx...')` statement to parse the specific execution plan. +- `BINARY_PLAN`: The original execution plan encoded in binary format. If there are multiple statements, the plan of only one statement is taken. Execute the [`SELECT tidb_decode_binary_plan('xxx...')`](/functions-and-operators/tidb-functions.md#tidb_decode_binary_plan) statement to parse the specific execution plan. - `PLAN_CACHE_HITS`: The total number of times that SQL statements of this category hit the plan cache. - `PLAN_IN_CACHE`: Indicates whether the previous execution of SQL statements of this category hit the plan cache. @@ -402,6 +402,16 @@ Transaction-related fields: - `AVG_AFFECTED_ROWS`: The average number of rows affected. - `PREV_SAMPLE_TEXT`: When the current SQL statement is `COMMIT`, `PREV_SAMPLE_TEXT` is the previous statement to `COMMIT`. In this case, SQL statements are grouped by the digest and `prev_sample_text`. This means that `COMMIT` statements with different `prev_sample_text` are grouped to different rows. When the current SQL statement is not `COMMIT`, the `PREV_SAMPLE_TEXT` field is an empty string. +Fields related to Resource Control: + +- `AVG_REQUEST_UNIT_WRITE`: the average number of write RUs consumed by SQL statements. +- `MAX_REQUEST_UNIT_WRITE`: the maximum number of write RUs consumed by SQL statements. +- `AVG_REQUEST_UNIT_READ`: the average number of read RUs consumed by SQL statements. +- `MAX_REQUEST_UNIT_READ`: the maximum number of read RUs consumed by SQL statements. +- `AVG_QUEUED_RC_TIME`: the average waiting time for available RU when executing SQL statements. +- `MAX_QUEUED_RC_TIME`: the maximum waiting time for available RU when executing SQL statements. +- `RESOURCE_GROUP`: the resource group bound to SQL statements. + ### `statements_summary_evicted` fields description - `BEGIN_TIME`: Records the starting time. diff --git a/statistics.md b/statistics.md index e22fa955770c7..d8c5aa3169e4e 100644 --- a/statistics.md +++ b/statistics.md @@ -1,7 +1,6 @@ --- title: Introduction to Statistics summary: Learn how the statistics collect table-level and column-level information. -aliases: ['/docs/dev/statistics/','/docs/dev/reference/performance/statistics/'] --- # Introduction to Statistics @@ -12,7 +11,7 @@ TiDB uses statistics to decide [which index to choose](/choose-index.md). The `tidb_analyze_version` variable controls the statistics collected by TiDB. Currently, two versions of statistics are supported: `tidb_analyze_version = 1` and `tidb_analyze_version = 2`. -- For TiDB Self-Hosted, the default value of this variable changes from `1` to `2` starting from v5.3.0. +- For TiDB Self-Managed, the default value of this variable changes from `1` to `2` starting from v5.3.0. - For TiDB Cloud, the default value of this variable changes from `1` to `2` starting from v6.5.0. - If your cluster is upgraded from an earlier version, the default value of `tidb_analyze_version` does not change after the upgrade. @@ -115,13 +114,13 @@ You can perform full collection using the following syntax. Before v5.3.0, TiDB uses the reservoir sampling method to collect statistics. Since v5.3.0, the TiDB Version 2 statistics uses the Bernoulli sampling method to collect statistics by default. To re-use the reservoir sampling method, you can use the `WITH NUM SAMPLES` statement. -The current sampling rate is calculated based on an adaptive algorithm. When you can observe the number of rows in a table using [`SHOW STATS_META`](/sql-statements/sql-statement-show-stats-meta.md), you can use this number of rows to calculate the sampling rate corresponding to 100,000 rows. If you cannot observe this number, you can use the `TABLE_KEYS` column in the [`TABLE_STORAGE_STATS`](/information-schema/information-schema-table-storage-stats.md) table as another reference to calculate the sampling rate. +The current sampling rate is calculated based on an adaptive algorithm. When you can observe the number of rows in a table using [`SHOW STATS_META`](/sql-statements/sql-statement-show-stats-meta.md), you can use this number of rows to calculate the sampling rate corresponding to 100,000 rows. If you cannot observe this number, you can use the sum of all the values in the `APPROXIMATE_KEYS` column in the results of [`SHOW TABLE REGIONS`](/sql-statements/sql-statement-show-table-regions.md) of the table as another reference to calculate the sampling rate. > **Note:** > -> Normally, `STATS_META` is more credible than `TABLE_KEYS`. However, after importing data through the methods like [TiDB Lightning](https://docs.pingcap.com/tidb/stable/tidb-lightning-overview), the result of `STATS_META` is `0`. To handle this situation, you can use `TABLE_KEYS` to calculate the sampling rate when the result of `STATS_META` is much smaller than the result of `TABLE_KEYS`. +> Normally, `STATS_META` is more credible than `APPROXIMATE_KEYS`. However, after importing data through the methods like [TiDB Lightning physical import mode](/tidb-lightning/tidb-lightning-physical-import-mode.md), the result of `STATS_META` is `0`. To handle this situation, you can use `APPROXIMATE_KEYS` to calculate the sampling rate when the result of `STATS_META` is much smaller than the result of `APPROXIMATE_KEYS`. @@ -129,7 +128,7 @@ The current sampling rate is calculated based on an adaptive algorithm. When you > **Note:** > -> Normally, `STATS_META` is more credible than `TABLE_KEYS`. However, after importing data through TiDB Cloud console (see [Import Sample Data](/tidb-cloud/import-sample-data.md)), the result of `STATS_META` is `0`. To handle this situation, you can use `TABLE_KEYS` to calculate the sampling rate when the result of `STATS_META` is much smaller than the result of `TABLE_KEYS`. +> Normally, `STATS_META` is more credible than `APPROXIMATE_KEYS`. However, after importing data through TiDB Cloud console (see [Import Sample Data](/tidb-cloud/import-sample-data.md)), the result of `STATS_META` is `0`. To handle this situation, you can use `APPROXIMATE_KEYS` to calculate the sampling rate when the result of `STATS_META` is much smaller than the result of `APPROXIMATE_KEYS`. @@ -408,15 +407,20 @@ The relationships of the relevant system variables are shown below: #### `tidb_build_stats_concurrency` -When you run the `ANALYZE` statement, the task is divided into multiple small tasks. Each task only works on statistics of one column or index. You can use the [`tidb_build_stats_concurrency`](/system-variables.md#tidb_build_stats_concurrency) variable to control the number of simultaneous small tasks. The default value is `2`. The default value is `4` for v7.4.0 and earlier versions. +This variable controls the concurrency for building statistics during manual `ANALYZE`, such as the number of table or partition analysis tasks that can be processed simultaneously. The default value is `2`. The default value is `4` for v7.4.0 and earlier versions. #### `tidb_build_sampling_stats_concurrency` -When analyzing ordinary columns, you can use [`tidb_build_sampling_stats_concurrency`](/system-variables.md#tidb_build_sampling_stats_concurrency-new-in-v750) to control the concurrency of executing sampling tasks. The default value is `2`. +This variable controls the following aspects of `ANALYZE` concurrency: + +- The concurrency for merging samples collected from different Regions. +- The concurrency for collecting statistics on special indexes (such as indexes on generated virtual columns), for example, the number of indexes that TiDB can concurrently collect statistics for. + +The default value is `2`. #### `tidb_analyze_partition_concurrency` -When running the `ANALYZE` statement, you can use [`tidb_analyze_partition_concurrency`](/system-variables.md#tidb_analyze_partition_concurrency) to control the concurrency of reading and writing statistics for a partitioned table. The default value is `2`. The default value is `1` for v7.4.0 and earlier versions. +This variable controls the concurrency for saving `ANALYZE` results (writing TopN and histograms to system tables). The default value is `2`. The default value is `1` for v7.4.0 and earlier versions. #### `tidb_distsql_scan_concurrency` @@ -457,7 +461,13 @@ The `ANALYZE` configuration persistence feature is disabled by default. To enabl You can use this feature to record the persistence configurations specified in the `ANALYZE` statement when executing the statement manually. Once recorded, the next time TiDB automatically updates statistics or you manually collect statistics without specifying these configuration, TiDB will collect statistics according to the recorded configurations. -When you manually execute the `ANALYZE` statement multiple times with persistence configurations specified, TiDB overwrites the previously recorded persistent configuration using the new configurations specified by the latest `ANALYZE` statement. +To query the configuration persisted on a specific table used for auto analyze operations, you can use the following SQL statement: + +```sql +SELECT sample_num, sample_rate, buckets, topn, column_choice, column_ids FROM mysql.analyze_options opt JOIN information_schema.tables tbl ON opt.table_id = tbl.tidb_table_id WHERE tbl.table_schema = '{db_name}' AND tbl.table_name = '{table_name}'; +``` + +TiDB will overwrite the previously recorded persistent configuration using the new configurations specified by the latest `ANALYZE` statement. For example, if you run `ANALYZE TABLE t WITH 200 TOPN;`, it will set the top 200 values in the `ANALYZE` statement. Subsequently, executing `ANALYZE TABLE t WITH 0.1 SAMPLERATE;` will set both the top 200 values and a sampling rate of 0.1 for auto `ANALYZE` statements, similar to `ANALYZE TABLE t WITH 200 TOPN, 0.1 SAMPLERATE;`. #### Disable ANALYZE configuration persistence @@ -649,7 +659,7 @@ Currently, the `SHOW STATS_BUCKETS` statement returns the following 11 columns: | `repeats` | The occurrence number of the maximum value | | `lower_bound` | The minimum value | | `upper_bound` | The maximum value | -| `ndv` | The number of different values in the bucket. When `tidb_analyze_version` = `1`, `ndv` is always `0`, which has no actual meaning. | +| `Ndv` | The number of distinct values in the bucket. This field is deprecated and always shows `0` due to its inaccurate value. | ### Top-N information @@ -707,7 +717,7 @@ The preceding statement only deletes GlobalStats generated in dynamic pruning mo > **Note:** > -> Loading statistics is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> Loading statistics is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. By default, depending on the size of column statistics, TiDB loads statistics differently as follows: @@ -726,7 +736,7 @@ After enabling the synchronously loading statistics feature, you can further con - To specify the maximum number of columns that the synchronously loading statistics feature can process concurrently, modify the value of the [`stats-load-concurrency`](/tidb-configuration-file.md#stats-load-concurrency-new-in-v540) option in the TiDB configuration file. The default value is `5`. - To specify the maximum number of column requests that the synchronously loading statistics feature can cache, modify the value of the [`stats-load-queue-size`](/tidb-configuration-file.md#stats-load-queue-size-new-in-v540) option in the TiDB configuration file. The default value is `1000`. -During TiDB startup, SQL statements executed before the initial statistics are fully loaded might have suboptimal execution plans, thus causing performance issues. To avoid such issues, TiDB v7.1.0 introduces the configuration parameter [`force-init-stats`](/tidb-configuration-file.md#force-init-stats-new-in-v710). With this option, you can control whether TiDB provides services only after statistics initialization has been finished during startup. Starting from v7.2.0, this parameter is enabled by default. +During TiDB startup, SQL statements executed before the initial statistics are fully loaded might have suboptimal execution plans, thus causing performance issues. To avoid such issues, TiDB v7.1.0 introduces the configuration parameter [`force-init-stats`](/tidb-configuration-file.md#force-init-stats-new-in-v657-and-v710). With this option, you can control whether TiDB provides services only after statistics initialization has been finished during startup. Starting from v7.2.0, this parameter is enabled by default. Starting from v7.1.0, TiDB introduces [`lite-init-stats`](/tidb-configuration-file.md#lite-init-stats-new-in-v710) for lightweight statistics initialization. @@ -801,10 +811,6 @@ LOAD STATS 'file_name' ## Lock statistics -> **Warning:** -> -> Locking statistics is an experimental feature for the current version. It is not recommended to use it in the production environment. - Starting from v6.5.0, TiDB supports locking statistics. After the statistics of a table or a partition are locked, the statistics of the table cannot be modified and the `ANALYZE` statement cannot be executed on the table. For example: Create table `t`, and insert data into it. When the statistics of table `t` are not locked, the `ANALYZE` statement can be successfully executed. diff --git a/status-variables.md b/status-variables.md new file mode 100644 index 0000000000000..efd0c8a10be00 --- /dev/null +++ b/status-variables.md @@ -0,0 +1,104 @@ +--- +title: Server Status Variables +summary: Use status variables to see the system and session status +--- + +# Server Status Variables + +Server status variables provide information about the global status of the server and the status of the current session in TiDB. Most of these variables are designed to be compatible with MySQL. + +You can retrieve the global status using the [SHOW GLOBAL STATUS](/sql-statements/sql-statement-show-status.md) command, and the status of the current session using the [SHOW SESSION STATUS](/sql-statements/sql-statement-show-status.md) command. + +Additionally, the [FLUSH STATUS](/sql-statements/sql-statement-flush-status.md) command is supported for MySQL compatibility. + +## Variable reference + +### Ssl_cipher + +- Scope: SESSION | GLOBAL +- Type: String +- TLS Cipher that is in use. + +### Ssl_cipher_list + +- Scope: SESSION | GLOBAL +- Type: String +- The list of TLS Ciphers that the server supports. + +### Ssl_server_not_after + +- Scope: SESSION | GLOBAL +- Type: Date +- The expiration date of the X.509 certificate of the server that is used for TLS connections. + +### Ssl_server_not_before + +- Scope: SESSION | GLOBAL +- Type: String +- The start date of the X.509 certificate of the server that is used for TLS connections. + +### Ssl_verify_mode + +- Scope: SESSION | GLOBAL +- Type: Integer +- The TLS verification mode bitmask. + +### Ssl_version + +- Scope: SESSION | GLOBAL +- Type: String +- The version of the TLS protocol that is used + +### Uptime + +- Scope: SESSION | GLOBAL +- Type: Integer +- Uptime of the server in seconds. + +### ddl_schema_version + +- Scope: SESSION | GLOBAL +- Type: Integer +- The version of the DDL schema that is used. + +### last_plan_binding_update_time New in v5.2.0 + +- Scope: SESSION +- Type: Timestamp +- The time and date of the last plan binding update. + +### server_id + +- Scope: SESSION | GLOBAL +- Type: String +- The UUID of the server. + +### tidb_gc_last_run_time + +- Scope: SESSION | GLOBAL +- Type: String +- The timestamp of the last run of [GC](/garbage-collection-overview.md). + +### tidb_gc_leader_desc + +- Scope: SESSION | GLOBAL +- Type: String +- Information about [GC](/garbage-collection-overview.md) leader, including the hostname and process id (pid). + +### tidb_gc_leader_lease + +- Scope: SESSION | GLOBAL +- Type: String +- The timestamp of the [GC](/garbage-collection-overview.md) lease. + +### tidb_gc_leader_uuid + +- Scope: SESSION | GLOBAL +- Type: String +- The UUID of the [GC](/garbage-collection-overview.md) leader. + +### tidb_gc_safe_point + +- Scope: SESSION | GLOBAL +- Type: String +- The timestamp of the [GC](/garbage-collection-overview.md) safe point. diff --git a/storage-engine/rocksdb-overview.md b/storage-engine/rocksdb-overview.md index 8ada230088a7d..c5ea5676aa8de 100644 --- a/storage-engine/rocksdb-overview.md +++ b/storage-engine/rocksdb-overview.md @@ -22,7 +22,7 @@ As the storage engine of TiKV, RocksDB is used to store Raft logs and user data. * raft CF: Store metadata of each Region. It occupies only a very small amount of space, and users do not need to care. * lock CF: Store the pessimistic lock of pessimistic transactions and the Prewrite lock for distributed transactions. After the transaction is committed, the corresponding data in lock CF is deleted quickly. Therefore, the size of data in lock CF is usually very small (less than 1 GB). If the data in lock CF increases a lot, it means that a large number of transactions are waiting to be committed, and that the system meets a bug or failure. -* write CF: Store the user's real written data and MVCC metadata (the start timestamp and commit timestamp of the transaction to which the data belongs). When the user writes a row of data, it is stored in the write CF if the data length is less than 255 bytes. Otherwise, it is stored in the default CF. In TiDB, the secondary index only occupies the space of write CF, since the value stored in the non-unique index is empty and the value stored in the unique index is the primary key index. +* write CF: Store the user's real written data and MVCC metadata (the start timestamp and commit timestamp of the transaction to which the data belongs). When the user writes a row of data, it is stored in the write CF if the data length is less than or equal to 255 bytes. Otherwise, it is stored in the default CF. In TiDB, the secondary index only occupies the space of write CF, since the value stored in the non-unique index is empty and the value stored in the unique index is the primary key index. * default CF: Store data longer than 255 bytes. ## RocksDB memory usage diff --git a/storage-engine/titan-configuration.md b/storage-engine/titan-configuration.md index b5bb12431c3cd..5cf9b9b0657ce 100644 --- a/storage-engine/titan-configuration.md +++ b/storage-engine/titan-configuration.md @@ -118,7 +118,11 @@ To disable Titan, you can configure the `rocksdb.defaultcf.titan.blob-run-mode` - When the option is set to `read-only`, all newly written values are written into RocksDB, regardless of the value size. - When the option is set to `fallback`, all newly written values are written into RocksDB, regardless of the value size. Also, all compacted values stored in the Titan blob file are automatically moved back to RocksDB. -To fully disable Titan for all existing and future data, you can follow these steps: +To disable Titan for all existing and future data, you can take the following steps. Note that you can skip Step 2 because it greatly affects online traffic performance. In fact even without Step 2, the data compaction consumes extra I/O and CPU resources when it moves data from Titan to RocksDB, and performance will degrade (sometimes as much as 50%) when TiKV I/O or CPU resources are limited. + +> **Warning:** +> +> When disabling Titan for TiDB versions earlier than v8.5.0, it is not recommended to modify the TiKV configuration item [`rocksdb.titan.enabled`](/tikv-configuration-file.md#enabled) to `false`, as this might cause TiKV to crash. Following Step 1 is sufficient to disable Titan. 1. Update the configuration of the TiKV nodes you wish to disable Titan for. You can update configuration in two methods: @@ -139,13 +143,6 @@ To fully disable Titan for all existing and future data, you can follow these st 3. After the compaction is finished, you should wait for the **Blob file count** metrics under **TiKV-Details**/**Titan - kv** to decrease to `0`. -4. Update the configuration of these TiKV nodes to disable Titan. - - ```toml - [rocksdb.titan] - enabled = false - ``` - ## Level Merge (experimental) In TiKV 4.0, [Level Merge](/storage-engine/titan-overview.md#level-merge), a new algorithm, is introduced to improve the performance of range query and to reduce the impact of Titan GC on the foreground write operations. You can enable Level Merge using the following option: diff --git a/styles/PingCAP/Ambiguous.yml b/styles/PingCAP/Ambiguous.yml index 54c294a609caa..8b457cdcec2ae 100644 --- a/styles/PingCAP/Ambiguous.yml +++ b/styles/PingCAP/Ambiguous.yml @@ -1,7 +1,6 @@ extends: existence message: Consider using a clearer word than '%s' because it may cause confusion. level: suggestion -code: false ignorecase: true tokens: - a lot diff --git a/styles/Vocab/PingCAP/accept.txt b/styles/config/vocabularies/PingCAP/accept.txt similarity index 100% rename from styles/Vocab/PingCAP/accept.txt rename to styles/config/vocabularies/PingCAP/accept.txt diff --git a/styles/Vocab/PingCAP/reject.txt b/styles/config/vocabularies/PingCAP/reject.txt similarity index 100% rename from styles/Vocab/PingCAP/reject.txt rename to styles/config/vocabularies/PingCAP/reject.txt diff --git a/styles/vocab.txt b/styles/vocab.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/subquery-optimization.md b/subquery-optimization.md index 49dde275c4327..5a4af0c2f79fe 100644 --- a/subquery-optimization.md +++ b/subquery-optimization.md @@ -24,7 +24,7 @@ By default, subqueries use `semi join` mentioned in [Understanding TiDB Executio In this case, `ALL` and `ANY` can be replaced by `MAX` and `MIN`. When the table is empty, the result of `MAX(EXPR)` and `MIN(EXPR)` is NULL. It works the same when the result of `EXPR` contains `NULL`. Whether the result of `EXPR` contains `NULL` may affect the final result of the expression, so the complete rewrite is given in the following form: - `t.id < all (select s.id from s)` is rewritten as `t.id < min(s.id) and if(sum(s.id is null) != 0, null, true)` -- `t.id < any (select s.id from s)` is rewritten as `t.id < max(s.id) or if(sum(s.id is null) != 0, null, false)` +- `t.id > any (select s.id from s)` is rewritten as `t.id > max(s.id) or if(sum(s.id is null) != 0, null, false)` ## `... != ANY (SELECT ... FROM ...)` diff --git a/support.md b/support.md index b9f13295279f4..cb43c593fd78e 100644 --- a/support.md +++ b/support.md @@ -18,7 +18,7 @@ If you encounter a problem when you use TiDB, you can reach out for support from + Seek help from the TiDB community: - - The [TiDB Forum](https://ask.pingcap.com/) + - [TiDB Community](https://ask.pingcap.com/) - [Discord channels](https://discord.gg/DQZ2dy3cuc?utm_source=doc) - Slack channels: [#everyone](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=docs) (English), [#tidb-japan](https://slack.tidb.io/invite?team=tidb-community&channel=tidb-japan&ref=docs) (Japanese) - [Stack Overflow](https://stackoverflow.com/questions/tagged/tidb) (questions tagged with #tidb) @@ -30,4 +30,4 @@ If you encounter a problem when you use TiDB, you can reach out for support from + Learn TiDB's implementation and design - [TiDB development guide](https://pingcap.github.io/tidb-dev-guide/) - - [TiDB Internals forum](https://internals.tidb.io/) + - [Discussions on GitHub](https://github.com/orgs/pingcap/discussions) diff --git a/sync-diff-inspector/route-diff.md b/sync-diff-inspector/route-diff.md index 4b0854acd0c39..e017df0f5f3b3 100644 --- a/sync-diff-inspector/route-diff.md +++ b/sync-diff-inspector/route-diff.md @@ -1,14 +1,13 @@ --- title: Data Check for Tables with Different Schema or Table Names summary: Learn the data check for different database names or table names. -aliases: ['/docs/dev/sync-diff-inspector/route-diff/','/docs/dev/reference/tools/sync-diff-inspector/route-diff/'] --- # Data Check for Tables with Different Schema or Table Names When using replication tools such as [TiDB Data Migration](/dm/dm-overview.md), you can set `route-rules` to replicate data to a specified table in the downstream. sync-diff-inspector enables you to verify tables with different schema names or table names by setting `rules`. -The following is a simple configuration example. To learn the complete configuration, refer to [Sync-diff-inspector User Guide](/sync-diff-inspector/sync-diff-inspector-overview.md). +The following is a simple configuration example. To learn the complete configuration, refer to [sync-diff-inspector User Guide](/sync-diff-inspector/sync-diff-inspector-overview.md). ```toml ######################### Datasource config ######################### @@ -58,6 +57,135 @@ target-schema = "test_2" # The name of the schema in the target database target-table = "t_2" # The name of the target table ``` -## Note +## The initialization of table routers and some examples -If `test_2`.`t_2` exists in the upstream database, the downstream database also compares this table. \ No newline at end of file +### The initialization of table routers + +- If a `target-schema/target-table` table named `schema.table` exists in the rules, the behavior of sync-diff-inspector is as follows: + + - If there is a rule that matches `schema.table` to `schema.table`, sync-diff-inspector does nothing. + - If there is no rule that matches `schema.table` to `schema.table`, sync-diff-inspector will add a new rule `schema.table -> _no__exists__db_._no__exists__table_` to the table router. After that, sync-diff-inspector will treat the table `schema.table` as the table `_no__exists__db_._no__exists__table_`. + +- If `target-schema` exists only in the rules as follows: + + ```toml + [routes.rule1] + schema-pattern = "schema_2" # the schema to match. Support wildcard characters * and ? + target-schema = "schema" # the target schema + ``` + + - If there is no schema `schema` in the upstream, sync-diff-inspector does nothing. + - If there is a schema `schema` in the upstream, and a rule matches the schema, sync-diff-inspector does nothing. + - If there is a schema `schema` in the upstream, but no rule matches the schema, sync-diff-inspector will add a new rule `schema -> _no__exists__db_` to the table router. After that, sync-diff-inspector will treat the table `schema` as the table `_no__exists__db_`. + +- If `target-schema.target-table` does not exist in the rules, sync-diff-inspector will add a rule to match `target-schema.target-table` to `target-schema.target-table` to make it case-insensitive, because the table router is case-insensitive. + +### Examples + +Suppose there are seven tables in the upstream cluster: + +- `inspector_mysql_0.tb_emp1` +- `Inspector_mysql_0.tb_emp1` +- `inspector_mysql_0.Tb_emp1` +- `inspector_mysql_1.tb_emp1` +- `Inspector_mysql_1.tb_emp1` +- `inspector_mysql_1.Tb_emp1` +- `Inspector_mysql_1.Tb_emp1` + +In the configuration example, the upstream cluster has a rule `Source.rule1`, and the target table is `inspector_mysql_1.tb_emp1`. + +#### Example 1 + +If the configuration is as follows: + +```toml +[Source.rule1] +schema-pattern = "inspector_mysql_0" +table-pattern = "tb_emp1" +target-schema = "inspector_mysql_1" +target-table = "tb_emp1" +``` + +The routing results will be as follows: + +- `inspector_mysql_0.tb_emp1` is routed to `inspector_mysql_1.tb_emp1` +- `Inspector_mysql_0.tb_emp1` is routed to `inspector_mysql_1.tb_emp1` +- `inspector_mysql_0.Tb_emp1` is routed to `inspector_mysql_1.tb_emp1` +- `inspector_mysql_1.tb_emp1` is routed to `_no__exists__db_._no__exists__table_` +- `Inspector_mysql_1.tb_emp1` is routed to `_no__exists__db_._no__exists__table_` +- `inspector_mysql_1.Tb_emp1` is routed to `_no__exists__db_._no__exists__table_` +- `Inspector_mysql_1.Tb_emp1` is routed to `_no__exists__db_._no__exists__table_` + +#### Example 2 + +If the configuration is as follows: + +```toml +[Source.rule1] +schema-pattern = "inspector_mysql_0" +target-schema = "inspector_mysql_1" +``` + +The routing results will be as follows: + +- `inspector_mysql_0.tb_emp1` is routed to `inspector_mysql_1.tb_emp1` +- `Inspector_mysql_0.tb_emp1` is routed to `inspector_mysql_1.tb_emp1` +- `inspector_mysql_0.Tb_emp1` is routed to `inspector_mysql_1.Tb_emp1` +- `inspector_mysql_1.tb_emp1` is routed to `_no__exists__db_._no__exists__table_` +- `Inspector_mysql_1.tb_emp1` is routed to `_no__exists__db_._no__exists__table_` +- `inspector_mysql_1.Tb_emp1` is routed to `_no__exists__db_._no__exists__table_` +- `Inspector_mysql_1.Tb_emp1` is routed to `_no__exists__db_._no__exists__table_` + +#### Example 3 + +If the configuration is as follows: + +```toml +[Source.rule1] +schema-pattern = "other_schema" +target-schema = "other_schema" +``` + +The routing results will be as follows: + +- `inspector_mysql_0.tb_emp1` is routed to `inspector_mysql_0.tb_emp1` +- `Inspector_mysql_0.tb_emp1` is routed to `Inspector_mysql_0.tb_emp1` +- `inspector_mysql_0.Tb_emp1` is routed to `inspector_mysql_0.Tb_emp1` +- `inspector_mysql_1.tb_emp1` is routed to `inspector_mysql_1.tb_emp1` +- `Inspector_mysql_1.tb_emp1` is routed to `inspector_mysql_1.tb_emp1` +- `inspector_mysql_1.Tb_emp1` is routed to `inspector_mysql_1.tb_emp1` +- `Inspector_mysql_1.Tb_emp1` is routed to `inspector_mysql_1.tb_emp1` + +#### Example 4 + +If the configuration is as follows: + +```toml +[Source.rule1] +schema-pattern = "inspector_mysql_?" +table-pattern = "tb_emp1" +target-schema = "inspector_mysql_1" +target-table = "tb_emp1" +``` + +The routing results will be as follows: + +- `inspector_mysql_0.tb_emp1` is routed to `inspector_mysql_1.tb_emp1` +- `Inspector_mysql_0.tb_emp1` is routed to `inspector_mysql_1.tb_emp1` +- `inspector_mysql_0.Tb_emp1` is routed to `inspector_mysql_1.tb_emp1` +- `inspector_mysql_1.tb_emp1` is routed to `inspector_mysql_1.tb_emp1` +- `Inspector_mysql_1.tb_emp1` is routed to `inspector_mysql_1.tb_emp1` +- `inspector_mysql_1.Tb_emp1` is routed to `inspector_mysql_1.tb_emp1` +- `Inspector_mysql_1.Tb_emp1` is routed to `inspector_mysql_1.tb_emp1` + +#### Example 5 + +If you do not set any rules, the routing results will be as follows: + +- `inspector_mysql_0.tb_emp1` is routed to `inspector_mysql_0.tb_emp1` +- `Inspector_mysql_0.tb_emp1` is routed to `Inspector_mysql_0.tb_emp1` +- `inspector_mysql_0.Tb_emp1` is routed to `inspector_mysql_0.Tb_emp1` +- `inspector_mysql_1.tb_emp1` is routed to `inspector_mysql_1.tb_emp1` +- `Inspector_mysql_1.tb_emp1` is routed to `inspector_mysql_1.tb_emp1` +- `inspector_mysql_1.Tb_emp1` is routed to `inspector_mysql_1.tb_emp1` +- `Inspector_mysql_1.Tb_emp1` is routed to `inspector_mysql_1.tb_emp1` diff --git a/sync-diff-inspector/shard-diff.md b/sync-diff-inspector/shard-diff.md index a8fefc2a2bb9d..22738a3e94e27 100644 --- a/sync-diff-inspector/shard-diff.md +++ b/sync-diff-inspector/shard-diff.md @@ -1,7 +1,6 @@ --- title: Data Check in the Sharding Scenario summary: Learn the data check in the sharding scenario. -aliases: ['/docs/dev/sync-diff-inspector/shard-diff/','/docs/dev/reference/tools/sync-diff-inspector/shard-diff/'] --- # Data Check in the Sharding Scenario diff --git a/sync-diff-inspector/sync-diff-inspector-overview.md b/sync-diff-inspector/sync-diff-inspector-overview.md index 56a2dd8937f37..c32174029ffb5 100644 --- a/sync-diff-inspector/sync-diff-inspector-overview.md +++ b/sync-diff-inspector/sync-diff-inspector-overview.md @@ -1,7 +1,6 @@ --- title: sync-diff-inspector User Guide summary: Use sync-diff-inspector to compare data and repair inconsistent data. -aliases: ['/docs/dev/sync-diff-inspector/sync-diff-inspector-overview/','/docs/dev/reference/tools/sync-diff-inspector/overview/'] --- # sync-diff-inspector User Guide @@ -38,16 +37,16 @@ This guide introduces the key features of sync-diff-inspector and describes how ## Database privileges for sync-diff-inspector -sync-diff-inspector needs to obtain the information of table schema and to query data. The required database privileges are as follows: +To access table schemas and query data, sync-diff-inspector requires specific database privileges. Grant the following privileges on both the upstream and downstream databases: -* Upstream database - - `SELECT` (checks data for comparison) - - `SHOW_DATABASES` (views database name) - - `RELOAD` (views table schema) -* Downstream database - - `SELECT` (checks data for comparison) - - `SHOW_DATABASES` (views database name) - - `RELOAD` (views table schema) +- `SELECT`: required to compare data. +- `RELOAD`: required to view table schemas. +- `PROCESS`: required when both the upstream and downstream are TiDB clusters. It is used to query the `INFORMATION_SCHEMA.CLUSTER_INFO` table. + +> **Note**: +> +> - **DO NOT** grant the [`SHOW DATABASES`](/sql-statements/sql-statement-show-databases.md) privilege on all databases (`*.*`). Otherwise, sync-diff-inspector will attempt to access inaccessible databases, which causes errors. +> - For MySQL data sources, ensure that the [`skip_show_database`](https://dev.mysql.com/doc/refman/8.4/en/server-system-variables.html#sysvar_skip_show_database) system variable is set to `OFF`. If this variable is set to `ON`, the check might fail. ## Configuration file description @@ -73,6 +72,9 @@ check-thread-count = 4 # If enabled, SQL statements is exported to fix inconsistent tables. export-fix-sql = true +# Only compares the data instead of the table structure. This configuration item is an experimental feature. It is not recommended that you use it in the production environment. +check-data-only = false + # Only compares the table structure instead of the data. check-struct-only = false @@ -182,7 +184,7 @@ sync-diff-inspector sends progress information to `stdout` when running. Progres > > To ensure the display effect, keep the display window width above 80 characters. -```progress +``` A total of 2 tables need to be compared Comparing the table structure of ``sbtest`.`sbtest96`` ... equivalent @@ -193,7 +195,7 @@ _____________________________________________________________________________ Progress [==========================================================>--] 98% 193/200 ``` -```progress +``` A total of 2 tables need to be compared Comparing the table structure of ``sbtest`.`sbtest96`` ... equivalent @@ -245,7 +247,7 @@ The running sync-diff-inspector periodically (every 10 seconds) prints the progr After the check is finished, sync-diff-inspector outputs a report. It is located at `${output}/summary.txt`, and `${output}` is the value of `output-dir` in the `config.toml` file. -```summary +``` +---------------------+--------------------+----------------+---------+-----------+ | TABLE | STRUCTURE EQUALITY | DATA DIFF ROWS | UPCOUNT | DOWNCOUNT | +---------------------+--------------------+----------------+---------+-----------+ @@ -260,6 +262,8 @@ Average Speed: 113.277149MB/s - `RESULT`: Whether the check is completed. If you have configured `skip-non-existing-table = true`, the value of this column is `skipped` for tables that do not exist in the upstream or downstream - `STRUCTURE EQUALITY`: Checks whether the table structure is the same - `DATA DIFF ROWS`: `rowAdd`/`rowDelete`. Indicates the number of rows that need to be added/deleted to fix the table +- `UPCOUNT`: The number of rows in this table in the upstream data source +- `DOWNCOUNT`: The number of rows in this table in the downstream data source ### SQL statements to fix inconsistent data @@ -288,7 +292,8 @@ REPLACE INTO `sbtest`.`sbtest99`(`id`,`k`,`c`,`pad`) VALUES (3700000,2501808,'he ## Note - sync-diff-inspector consumes a certain amount of server resources when checking data. Avoid using sync-diff-inspector to check data during peak business hours. -- Before comparing the data in MySQL with that in TiDB, pay attention to the collation configuration of the tables. If the primary key or unique key is the `varchar` type and the collation configuration in MySQL differs from that in TiDB, the final check result might be incorrect because of the collation issue. You need to add collation to the sync-diff-inspector configuration file. +- Before comparing the data in MySQL with that in TiDB, check the character set and `collation` configuration of the tables. This is especially important when the primary key or unique key of a table is the `varchar` type. If collation rules differ between upstream and downstream databases, sorting issues might occur, leading to inaccurate verification results. For example, MySQL's default collation is case-insensitive, while TiDB's default collation is case-sensitive. This inconsistency might cause identical delete and insert records in the repair SQL. To avoid this issue, use the `index-fields` configuration to specify index columns that are not affected by case sensitivity. If you configure `collation` in the sync-diff-inspector configuration file and explicitly use the same collation for both upstream and downstream during chunk-based comparison, note that the order of index fields depends on the table's collation configuration. If the collations differ, one side might be unable to use the index. Additionally, if the character sets differ between upstream and downstream (for example, MySQL uses UTF-8 while TiDB uses UTF-8MB4), it is not possible to unify the collation configuration. +- If the primary key differs between upstream and downstream tables, sync-diff-inspector does not use the original primary key column to divide chunks. For example, when sharded tables in MySQL are merged into TiDB using a composite primary key that includes the original primary key and a shard key. In this case, configure the original primary key column using `index-fields` and set `check-data-only` to `true`. - sync-diff-inspector divides data into chunks first according to TiDB statistics and you need to guarantee the accuracy of the statistics. You can manually run the `analyze table {table_name}` command when the TiDB server's *workload is light*. - Pay special attention to `table-rules`. If you configure `schema-pattern="test1"`, `table-pattern = "t_1"`, `target-schema="test2"` and `target-table = "t_2"`, the `test1`.`t_1` schema in the source database and the `test2`.`t_2` schema in the target database are compared. Sharding is enabled by default in sync-diff-inspector, so if the source database has a `test2`.`t_2` table, the `test1`.`t_1` table and `test2`.`t_2` table in the source database serving as sharding are compared with the `test2`.`t_2` table in the target database. - The generated SQL file is only used as a reference for repairing data, and you need to confirm it before executing these SQL statements to repair data. diff --git a/system-variables.md b/system-variables.md index cf05f2082609d..2d4a321d9e82d 100644 --- a/system-variables.md +++ b/system-variables.md @@ -1,7 +1,6 @@ --- title: System Variables summary: Use system variables to optimize performance or alter running behavior. -aliases: ['/tidb/dev/tidb-specific-system-variables','/docs/dev/system-variables/','/docs/dev/reference/configuration/tidb-server/mysql-variables/', '/docs/dev/tidb-specific-system-variables/','/docs/dev/reference/configuration/tidb-server/tidb-specific-variables/'] --- # System Variables @@ -50,7 +49,7 @@ Starting from v7.4.0, you can temporarily modify the value of some `SESSION` var For more information about the `SET_VAR` hint, see [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value). -## Variable Reference +## Variable reference ### allow_auto_random_explicit_insert New in v4.0.3 @@ -257,7 +256,7 @@ For more information about the `SET_VAR` hint, see [SET_VAR](/optimizer-hints.md - Type: Integer - Default value: `1` - Range: `[1, 65535]` -- Controls the step size of `AUTO_INCREMENT` values to be allocated to a column. It is often used in combination with `auto_increment_offset`. +- Controls the step size of `AUTO_INCREMENT` values to be allocated to a column, and allocation rules for `AUTO_RANDOM` IDs. It is often used in combination with [`auto_increment_offset`](#auto_increment_offset). ### auto_increment_offset @@ -267,7 +266,7 @@ For more information about the `SET_VAR` hint, see [SET_VAR](/optimizer-hints.md - Type: Integer - Default value: `1` - Range: `[1, 65535]` -- Controls the initial offset of `AUTO_INCREMENT` values to be allocated to a column. This setting is often used in combination with `auto_increment_increment`. For example: +- Controls the initial offset of `AUTO_INCREMENT` values to be allocated to a column, and allocation rules for `AUTO_RANDOM` IDs. This setting is often used in combination with [`auto_increment_increment`](#auto_increment_increment). For example: ```sql mysql> CREATE TABLE t1 (a int not null primary key auto_increment); @@ -392,7 +391,7 @@ mysql> SELECT * FROM t1; > **Note:** > -> This variable is not supported on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is not supported on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential). @@ -443,7 +442,6 @@ mysql> SELECT * FROM t1; - Type: Enumeration - Default value: `mysql_native_password` - Possible values: `mysql_native_password`, `caching_sha2_password`, `tidb_sm3_password`, `tidb_auth_token`, `authentication_ldap_sasl`, and `authentication_ldap_simple`. -- The `tidb_auth_token` authentication method is used only for the internal operation of TiDB Cloud. **DO NOT** set the variable to this value. - This variable sets the authentication method that the server advertises when the server-client connection is being established. - To authenticate using the `tidb_sm3_password` method, you can connect to TiDB using [TiDB-JDBC](https://github.com/pingcap/mysql-connector-j/tree/release/8.0-sm3). @@ -584,7 +582,7 @@ This variable is an alias for [`last_insert_id`](#last_insert_id). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential). - Scope: SESSION | GLOBAL - Persists to cluster: Yes @@ -647,7 +645,7 @@ This variable is an alias for [`last_insert_id`](#last_insert_id). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential). - Scope: SESSION | GLOBAL - Persists to cluster: Yes @@ -656,6 +654,7 @@ This variable is an alias for [`last_insert_id`](#last_insert_id). - Range: `[1024, 1073741824]` - The value should be an integer multiple of 1024. If the value is not divisible by 1024, a warning will be prompted and the value will be rounded down. For example, when the value is set to 1025, the actual value in TiDB is 1024. - The maximum packet size allowed by the server and the client in one transmission of packets. +- In the `SESSION` scope, this variable is read-only. - This variable is compatible with MySQL. ### password_history New in v6.5.0 @@ -687,11 +686,12 @@ This variable is an alias for [`last_insert_id`](#last_insert_id). - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): Yes - Default value: `UNSPECIFIED` -- Value options: `UNSPECIFIED`, `0`, `1` +- Value options: `UNSPECIFIED`, `0`, `1`, `2` - This variable is used to specify different versions of the MPP execution plan. After a version is specified, TiDB selects the specified version of the MPP execution plan. The meanings of the variable values are as follows: - - `UNSPECIFIED`: means unspecified. TiDB automatically selects the latest version `1`. + - `UNSPECIFIED`: means unspecified. TiDB automatically selects the latest version `2`. - `0`: compatible with all TiDB cluster versions. Features with the MPP version greater than `0` do not take effect in this mode. - `1`: new in v6.6.0, used to enable data exchange with compression on TiFlash. For details, see [MPP version and exchange data compression](/explain-mpp.md#mpp-version-and-exchange-data-compression). + - `2`: new in v7.3.0, used to provide more accurate error messages when MPP tasks encounter errors on TiFlash. ### password_reuse_interval New in v6.5.0 @@ -727,7 +727,7 @@ This variable is an alias for [`last_insert_id`](#last_insert_id). > **Note:** > -> The `max_execution_time` system variable currently only controls the maximum execution time for read-only SQL statements. The precision of the timeout value is roughly 100ms. This means the statement might not be terminated in accurate milliseconds as you specify. +> Before v6.4.0, the `max_execution_time` system variable takes effect on all types of statements. Starting from v6.4.0, this variable only controls the maximum execution time of read-only statements. The precision of the timeout value is roughly 100ms. This means the statement might not be terminated in exact milliseconds as you specify. @@ -778,7 +778,7 @@ mysql> SHOW GLOBAL VARIABLES LIKE 'max_prepared_stmt_count'; > **Note:** > -> This variable is not supported on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is not supported on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential). - Scope: GLOBAL - Persists to cluster: No, only applicable to the current TiDB instance that you are connecting to. @@ -790,7 +790,7 @@ mysql> SHOW GLOBAL VARIABLES LIKE 'max_prepared_stmt_count'; > **Note:** > -> This variable is not supported on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is not supported on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential). - Scope: GLOBAL - Persists to cluster: No, only applicable to the current TiDB instance that you are connecting to. @@ -831,13 +831,13 @@ mysql> SHOW GLOBAL VARIABLES LIKE 'max_prepared_stmt_count'; > **Note:** > -> Currently, this variable is not supported on [TiDB Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-dedicated). DO **NOT** enable this variable for TiDB Dedicated clusters. Otherwise, you might get SQL client connection failures. This restriction is a temporary control measure and will be resolved in a future release. +> Currently, this variable is not supported on [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated). DO **NOT** enable this variable for TiDB Cloud Dedicated clusters. Otherwise, you might get SQL client connection failures. This restriction is a temporary control measure and will be resolved in a future release. - Scope: GLOBAL - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Boolean -- Default value: `OFF` for TiDB Self-Hosted and [TiDB Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-dedicated), `ON` for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) +- Default value: `OFF` for TiDB Self-Managed and [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated), `ON` for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) @@ -853,12 +853,13 @@ mysql> SHOW GLOBAL VARIABLES LIKE 'max_prepared_stmt_count'; - Setting this variable to `ON` requires you to connect to TiDB from a session that has TLS enabled. This helps prevent lock-out scenarios when TLS is not configured correctly. - This setting was previously a `tidb.toml` option (`security.require-secure-transport`), but changed to a system variable starting from TiDB v6.1.0. +- For v7.5.1 or later v7.5 patch versions, when Security Enhanced Mode (SEM) is enabled, setting this variable to `ON` is prohibited to avoid potential connectivity issues for users. ### skip_name_resolve New in v5.2.0 > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential). - Scope: GLOBAL - Persists to cluster: Yes @@ -889,7 +890,7 @@ mysql> SHOW GLOBAL VARIABLES LIKE 'max_prepared_stmt_count'; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter). - Scope: SESSION | GLOBAL - Persists to cluster: Yes @@ -921,6 +922,12 @@ mysql> SHOW GLOBAL VARIABLES LIKE 'max_prepared_stmt_count'; - This feature is based on the similarly named [`sql_require_primary_key`](https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_sql_require_primary_key) in MySQL 8.0. - It is strongly recommended to enable this variable when using TiCDC. This is because replicating changes to a MySQL sink requires that tables have a primary key. + + +- If you enable this variable and are using TiDB Data Migration (DM) to migrate data, it is recommended that you add `sql_require_ primary_key` to the `session` part in the [DM Task Configuration File](/dm/task-configuration-file-full.md#task-configuration-file-template-advanced) and set it to `OFF`. Otherwise, it will cause DM to fail to create tasks. + + + ### sql_select_limit New in v4.0.2 - Scope: SESSION | GLOBAL @@ -1004,11 +1011,23 @@ mysql> SHOW GLOBAL VARIABLES LIKE 'max_prepared_stmt_count'; - Scope: SESSION | GLOBAL - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No +- Type: Integer - Default value: `4096` - Range: `[0, 9223372036854775807]` - Unit: Bytes - This variable is used to control the threshold at which the TiDB server prefers to send read requests to a replica in the same availability zone as the TiDB server when [`tidb_replica_read`](#tidb_replica_read-new-in-v40) is set to `closest-adaptive`. If the estimated result is higher than or equal to this threshold, TiDB prefers to send read requests to a replica in the same availability zone. Otherwise, TiDB sends read requests to the leader replica. +### tidb_allow_tiflash_cop New in v7.3.0 + +- Scope: SESSION | GLOBAL +- Persists to cluster: Yes +- Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No +- Type: Boolean +- Default value: `OFF` +- When TiDB pushes computation tasks down to TiFlash, there are three methods (or protocols) to choose from: Cop, BatchCop, and MPP. Compared to Cop and BatchCop, the MPP protocol is more mature and offers better task and resource management. Therefore, it is recommended to use the MPP protocol. + - `0` or `OFF`: the optimizer only generates plans using the TiFlash MPP protocol. + - `1` or `ON`: the optimizer determines whether to use the Cop, BatchCop, or MPP protocol to generate execution plans based on the cost estimation. + ### tidb_allow_batch_cop New in v4.0 - Scope: SESSION | GLOBAL @@ -1046,7 +1065,7 @@ mysql> SHOW GLOBAL VARIABLES LIKE 'max_prepared_stmt_count'; - Type: Boolean - Default value: `ON` - Controls whether to use the MPP mode of TiFlash to execute queries. The value options are as follows: - - `0` or `OFF`, which means that the MPP mode will not be used. + - `0` or `OFF`, which means that the MPP mode will not be used. For v7.3.0 or a later version, if you set the value of this variable to `0` or `OFF`, you also need to enable the [`tidb_allow_tiflash_cop`](/system-variables.md#tidb_allow_tiflash_cop-new-in-v730) variable. Otherwise, queries might return errors. - `1` or `ON`, which means that the optimizer determines whether to use the MPP mode based on the cost estimation (by default). MPP is a distributed computing framework provided by the TiFlash engine, which allows data exchange between nodes and provides high-performance, high-throughput SQL algorithms. For details about the selection of the MPP mode, refer to [Control whether to select the MPP mode](/tiflash/use-tiflash-mpp-mode.md#control-whether-to-select-the-mpp-mode). @@ -1069,15 +1088,11 @@ MPP is a distributed computing framework provided by the TiFlash engine, which a ### tidb_analyze_partition_concurrency -> **Warning:** -> -> The feature controlled by this variable is not fully functional in the current TiDB version. Do not change the default value. - - Scope: SESSION | GLOBAL - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Default value: `2`. The default value is `1` for v7.4.0 and earlier versions. -- This variable specifies the concurrency of reading and writing statistics for a partitioned table when TiDB analyzes the partitioned table. +- For manual and auto `ANALYZE`, this variable controls the concurrency for saving `ANALYZE` results, including writing TopN and histograms to system tables. ### tidb_analyze_version New in v5.1.0 @@ -1088,7 +1103,7 @@ MPP is a distributed computing framework provided by the TiFlash engine, which a - Default value: `2` - Range: `[1, 2]` - Controls how TiDB collects statistics. - - For TiDB Self-Hosted, the default value of this variable changes from `1` to `2` starting from v5.3.0. + - For TiDB Self-Managed, the default value of this variable changes from `1` to `2` starting from v5.3.0. - For TiDB Cloud, the default value of this variable changes from `1` to `2` starting from v6.5.0. - If your cluster is upgraded from an earlier version, the default value of `tidb_analyze_version` does not change after the upgrade. - For detailed introduction about this variable, see [Introduction to Statistics](/statistics.md). @@ -1199,7 +1214,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; - Type: Integer - Default value: `1` - Range: `[1, 256]` -- This variable is used to set the concurrency of executing the automatic update of statistics. +- This variable controls the concurrency for building statistics during auto `ANALYZE`, such as the number of table or partition analysis tasks that can be processed simultaneously. ### tidb_backoff_lock_fast @@ -1304,20 +1319,20 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; - Default value: `2`. The default value is `4` for v7.4.0 and earlier versions. - Range: `[1, 256]` - Unit: Threads -- This variable is used to set the concurrency of executing the `ANALYZE` statement. -- When the variable is set to a larger value, the execution performance of other queries is affected. +- This variable controls the concurrency for building statistics during manual `ANALYZE`, such as the number of table or partition analysis tasks that can be processed simultaneously. ### tidb_build_sampling_stats_concurrency New in v7.5.0 -- Scope: GLOBAL +- Scope: SESSION | GLOBAL - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Integer - Unit: Threads - Default value:`2` - Range: `[1, 256]` -- This variable is used to set the sampling concurrency in the `ANALYZE` process. -- When the variable is set to a larger value, the execution performance of other queries is affected. +- This variable controls the following aspects of `ANALYZE` concurrency: + - The concurrency for merging samples collected from different Regions. + - The concurrency for collecting statistics on special indexes (such as indexes on generated virtual columns), for example, the number of indexes that TiDB can concurrently collect statistics for. ### tidb_capture_plan_baselines New in v4.0 @@ -1333,7 +1348,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential). - Scope: SESSION - Persists to cluster: No @@ -1513,7 +1528,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential). - Scope: GLOBAL - Persists to cluster: Yes @@ -1528,8 +1543,9 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> - If you are using a [TiDB Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-dedicated) cluster, to improve the speed for index creation using this variable, make sure that your TiDB cluster is hosted on AWS and your TiDB node size is at least 8 vCPU. -> - For [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters, this variable is read-only. +> - If you are using a [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated) cluster, to improve the speed for index creation using this variable, make sure that your TiDB cluster is hosted on AWS and your TiDB node size is at least 8 vCPU. +> - For [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated) clusters with 4 vCPU, it is recommended to manually disable [`tidb_ddl_enable_fast_reorg`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) to prevent resource limitations from affecting cluster stability during index creation. Disabling this setting allows indexes to be created using transactions, which reduces the overall impact on the cluster. +> - For [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters, this variable is read-only. - Scope: GLOBAL - Persists to cluster: Yes @@ -1542,10 +1558,6 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; -> **Warning:** -> -> Currently, PITR recovery handles the indexes created by index acceleration during the log backup with extra processing to achieve compatibility. For details, see [Why is the acceleration of adding indexes feature incompatible with PITR?](/faq/backup-and-restore-faq.md#why-is-the-acceleration-of-adding-indexes-feature-incompatible-with-pitr). - > **Note:** > > * Index acceleration requires a [`temp-dir`](/tidb-configuration-file.md#temp-dir-new-in-v630) that is writable and has enough free space. If the `temp-dir` is unusable, TiDB falls back to non-accelerated index building. It is recommended to put the `temp-dir` on a SSD disk. @@ -1559,8 +1571,6 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Warning:** > > Currently, this feature is not fully compatible with [altering multiple columns or indexes in a single `ALTER TABLE` statement](/sql-statements/sql-statement-alter-table.md). When adding a unique index with the index acceleration, you need to avoid altering other columns or indexes in the same statement. -> -> Currently, PITR recovery handles the indexes created by index acceleration during the log backup with extra processing to achieve compatibility. For details, see [Why is the acceleration of adding indexes feature incompatible with PITR?](https://docs.pingcap.com/tidb/v7.0/backup-and-restore-faq#why-is-the-acceleration-of-adding-indexes-feature-incompatible-with-pitr). @@ -1570,9 +1580,9 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Default value: `OFF` -- This variable is used to control whether to enable the [TiDB backend task distributed execution framework](/tidb-distributed-execution-framework.md). After the framework is enabled, backend tasks such as DDL and import will be distributedly executed and completed by multiple TiDB nodes in the cluster. -- Starting from TiDB v7.1.0, the framework supports distributedly executing the [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) statement for partitioned tables. -- Starting from TiDB v7.2.0, the framework supports distributedly executing the [`IMPORT INTO`](https://docs.pingcap.com/tidb/v7.2/sql-statement-import-into) statement for import jobs of TiDB Self-Hosted. For TiDB Cloud, the `IMPORT INTO` statement is not applicable. +- This variable is used to control whether to enable the [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md). After the framework is enabled, the DXF tasks such as DDL and import will be distributedly executed and completed by multiple TiDB nodes in the cluster. +- Starting from TiDB v7.1.0, the DXF supports distributedly executing the [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) statement for partitioned tables. +- Starting from TiDB v7.2.0, the DXF supports distributedly executing the [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) statement for import jobs. - This variable is renamed from `tidb_ddl_distribute_reorg`. ### tidb_cloud_storage_uri New in v7.4.0 @@ -1585,29 +1595,16 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Default value: `""` - - - -- This variable is used to specify the Amazon S3 cloud storage URI to enable [Global Sort](/tidb-global-sort.md). After enabling the [distributed execution framework](/tidb-distributed-execution-framework.md), you can use the Global Sort feature by configuring the URI and pointing it to an appropriate cloud storage path with the necessary permissions to access the storage. For more details, see [Amazon S3 URI format](/external-storage-uri.md#amazon-s3-uri-format). -- The following statements can use the Global Sort feature. - - The [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) statement. - - The [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) statement for import jobs of TiDB Self-Hosted. For TiDB Cloud, the `IMPORT INTO` statement is not applicable. - - - - -- This variable is used to specify the cloud storage URI to enable [Global Sort](/tidb-global-sort.md). After enabling the [distributed execution framework](/tidb-distributed-execution-framework.md), you can use the Global Sort feature by configuring the URI and pointing it to an appropriate cloud storage path with the necessary permissions to access the storage. For more details, see [URI Formats of External Storage Services](https://docs.pingcap.com/tidb/stable/external-storage-uri). +- This variable is used to specify the Amazon S3 cloud storage URI to enable [Global Sort](/tidb-global-sort.md). After enabling the [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md), you can use the Global Sort feature by configuring the URI and pointing it to an appropriate cloud storage path with the necessary permissions to access the storage. For more details, see [Amazon S3 URI format](/external-storage-uri.md#amazon-s3-uri-format). - The following statements can use the Global Sort feature. - The [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) statement. - - The [`IMPORT INTO`](https://docs.pingcap.com/tidb/v7.2/sql-statement-import-into) statement for import jobs of TiDB Self-Hosted. For TiDB Cloud, the `IMPORT INTO` statement is not applicable. - - + - The [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) statement for import jobs. ### tidb_ddl_error_count_limit > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential). - Scope: GLOBAL - Persists to cluster: Yes @@ -1621,7 +1618,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential). - Scope: GLOBAL - Persists to cluster: Yes @@ -1629,13 +1626,13 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; - Type: Integer - Default value: `64` - Range: `[1, 256]` -- This variable controls the concurrency of [`FLASHBACK CLUSTER TO TIMESTAMP`](/sql-statements/sql-statement-flashback-to-timestamp.md). +- This variable controls the concurrency of [`FLASHBACK CLUSTER`](/sql-statements/sql-statement-flashback-cluster.md). ### tidb_ddl_reorg_batch_size > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential). - Scope: GLOBAL - Persists to cluster: Yes @@ -1645,14 +1642,14 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; - Range: `[32, 10240]` - Unit: Rows - This variable is used to set the batch size during the `re-organize` phase of the DDL operation. For example, when TiDB executes the `ADD INDEX` operation, the index data needs to backfilled by `tidb_ddl_reorg_worker_cnt` (the number) concurrent workers. Each worker backfills the index data in batches. - - If many updating operations such as `UPDATE` and `REPLACE` exist during the `ADD INDEX` operation, a larger batch size indicates a larger probability of transaction conflicts. In this case, you need to adjust the batch size to a smaller value. The minimum value is 32. - - If the transaction conflict does not exist, you can set the batch size to a large value (consider the worker count. See [Interaction Test on Online Workloads and `ADD INDEX` Operations](https://docs.pingcap.com/tidb/stable/online-workloads-and-add-index-operations) for reference). This can increase the speed of the backfilling data, but the write pressure on TiKV also becomes higher. + - If `tidb_ddl_enable_fast_reorg` is set to `OFF`, `ADD INDEX` is executed as a transaction. If there are many update operations such as `UPDATE` and `REPLACE` in the target columns during the `ADD INDEX` execution, a larger batch size indicates a larger probability of transaction conflicts. In this case, it is recommended that you set the batch size to a smaller value. The minimum value is 32. + - If the transaction conflict does not exist, or if `tidb_ddl_enable_fast_reorg` is set to `ON`, you can set the batch size to a large value. This makes data backfilling faster but also increases the write pressure on TiKV. For a proper batch size, you also need to refer to the value of `tidb_ddl_reorg_worker_cnt`. See [Interaction Test on Online Workloads and `ADD INDEX` Operations](https://docs.pingcap.com/tidb/dev/online-workloads-and-add-index-operations) for reference. ### tidb_ddl_reorg_priority > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential). - Scope: SESSION - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No @@ -1662,11 +1659,25 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; - This variable is used to set the priority of executing the `ADD INDEX` operation in the `re-organize` phase. - You can set the value of this variable to `PRIORITY_LOW`, `PRIORITY_NORMAL` or `PRIORITY_HIGH`. +### tidb_ddl_reorg_max_write_speed New in v6.5.12 and v7.5.5 + +- Scope: GLOBAL +- Persists to cluster: Yes +- Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No +- Type: String +- Default value: `0` +- Range: `[0, 1PiB]` +- This variable limits the write bandwidth for each TiKV node and only takes effect when index creation acceleration is enabled (controlled by the [`tidb_ddl_enable_fast_reorg`](#tidb_ddl_enable_fast_reorg-new-in-v630) variable). When the data size in your cluster is quite large (such as billions of rows), limiting the write bandwidth for index creation can effectively reduce the impact on application workloads. +- The default value `0` means no write bandwidth limit. +- You can specify the value of this variable either with a unit or without a unit. + - When you specify the value without a unit, the default unit is bytes per second. For example, `67108864` represents `64MiB` per second. + - When you specify the value with a unit, supported units include KiB, MiB, GiB, and TiB. For example, `'1GiB`' represents 1 GiB per second, and `'256MiB'` represents 256 MiB per second. + ### tidb_ddl_reorg_worker_cnt > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential). - Scope: GLOBAL - Persists to cluster: Yes @@ -1731,6 +1742,8 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; - Use a bigger value in OLAP scenarios, and a smaller value in OLTP scenarios. - For OLAP scenarios, the maximum value should not exceed the number of CPU cores of all the TiKV nodes. - If a table has a lot of partitions, you can reduce the variable value appropriately (determined by the size of the data to be scanned and the frequency of the scan) to avoid TiKV becoming out of memory (OOM). +- For a simple query with only a `LIMIT` clause, if the `LIMIT` value is less than 100000, the scan operation pushed down to TiKV treats the value of this variable as `1` to enhance execution efficiency. +- For the `SELECT MAX/MIN(col) FROM ...` query, if the `col` column has an index sorted in the same order required by the `MAX(col)` or `MIN(col)` function, TiDB will rewrite the query to `SELECT col FROM ... LIMIT 1` for processing, and the value of this variable will also be processed as `1`. For example, for `SELECT MIN(col) FROM ...`, if the `col` column has an ascending index, TiDB can quickly obtain the `MIN(col)` value by rewriting the query to `SELECT col FROM ... LIMIT 1` and directly reading the first row of the index. ### tidb_dml_batch_size @@ -1757,7 +1770,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential). - Scope: SESSION | GLOBAL - Persists to cluster: Yes @@ -1790,7 +1803,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential). - Scope: SESSION | GLOBAL - Persists to cluster: Yes @@ -1809,7 +1822,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential). - Scope: GLOBAL - Persists to cluster: Yes @@ -1888,7 +1901,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential). - Scope: GLOBAL - Persists to cluster: No, only applicable to the current TiDB instance that you are connecting to. @@ -2031,7 +2044,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential). - Scope: GLOBAL - Persists to cluster: Yes @@ -2067,7 +2080,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -2082,8 +2095,8 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Boolean -- Default value: `ON` -- This variable controls whether to enable historical statistics. The default value changes from `OFF` to `ON`, which means that historical statistics are enabled by default. +- Default value: `OFF`. Before v7.5.7, the default value is `ON`. +- This variable controls whether to enable historical statistics. The default value is `OFF`, which means that historical statistics are disabled by default. ### tidb_enable_historical_stats_for_capture @@ -2148,7 +2161,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -2441,7 +2454,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -2486,7 +2499,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -2600,7 +2613,7 @@ Query OK, 0 rows affected (0.09 sec) > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -2807,11 +2820,15 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified - This variable is used to change the default priority for statements executed on a TiDB server. A use case is to ensure that a particular user that is performing OLAP queries receives lower priority than users performing OLTP queries. - The default value `NO_PRIORITY` means that the priority for statements is not forced to change. +> **Note:** +> +> Starting from v6.6.0, TiDB supports [Resource Control](/tidb-resource-control.md). You can use this feature to execute SQL statements with different priorities in different resource groups. By configuring proper quotas and priorities for these resource groups, you can gain better scheduling control for SQL statements with different priorities. When resource control is enabled, statement priority will no longer take effect. It is recommended that you use [Resource Control](/tidb-resource-control.md) to manage resource usage for different SQL statements. + ### tidb_gc_concurrency New in v5.0 > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -2826,7 +2843,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -2837,16 +2854,12 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified ### tidb_gc_life_time New in v5.0 -> **Note:** -> -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). - - Scope: GLOBAL - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Duration - Default value: `10m0s` -- Range: `[10m0s, 8760h0m0s]` +- Range: `[10m0s, 8760h0m0s]` for TiDB Self-Managed and [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated), `[10m0s, 168h0m0s]` for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) - The time limit during which data is retained for each GC, in the format of Go Duration. When a GC happens, the current time minus this value is the safe point. > **Note:** @@ -2860,7 +2873,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -2875,7 +2888,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -2893,7 +2906,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -2969,7 +2982,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -2978,7 +2991,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified - Default value: `ON` - This variable controls whether to generate binary-encoded execution plans in slow logs and statement summaries. - When this variable is set to `ON`, you can view visual execution plans in TiDB Dashboard. Note that TiDB Dashboard only provides visual display for execution plans generated after this variable is enabled. -- You can execute the `SELECT tidb_decode_binary_plan('xxx...')` statement to parse the specific plan from a binary plan. +- You can execute the [`SELECT tidb_decode_binary_plan('xxx...')`](/functions-and-operators/tidb-functions.md#tidb_decode_binary_plan) statement to parse the specific plan from a binary plan. ### tidb_gogc_tuner_max_value New in v7.5.0 @@ -3004,7 +3017,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -3017,7 +3030,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: SESSION | GLOBAL - Persists to cluster: Yes @@ -3025,7 +3038,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified - Type: Boolean - Default value: `ON` - This variable controls the way commit TS is calculated for async commit. By default (with the `ON` value), the two-phase commit requests a new TS from the PD server and uses the TS to calculate the final commit TS. In this situation, linearizability is guaranteed for all the concurrent transactions. -- If you set this variable to `OFF`, the process of fetching TS from the PD server is skipped, with the cost that only causal consistency is guaranteed but not linearizability. For more details, see the blog post [Async Commit, the Accelerator for Transaction Commit in TiDB 5.0](https://en.pingcap.com/blog/async-commit-the-accelerator-for-transaction-commit-in-tidb-5-0/). +- If you set this variable to `OFF`, the process of fetching TS from the PD server is skipped, with the cost that only causal consistency is guaranteed but not linearizability. For more details, see the blog post [Async Commit, the Accelerator for Transaction Commit in TiDB 5.0](https://www.pingcap.com/blog/async-commit-the-accelerator-for-transaction-commit-in-tidb-5-0/). - For scenarios that require only causal consistency, you can set this variable to `OFF` to improve performance. ### tidb_hash_exchange_with_new_collation @@ -3207,13 +3220,13 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified - Default value: `32` - Range: `[1, 32]` - Unit: Rows -- This variable is used to set the number of rows for the initial chunk during the execution process. +- This variable is used to set the number of rows for the initial chunk during the execution process. The number of rows for a chunk directly affects the amount of memory required for a single query. You can roughly estimate the memory needed for a single chunk by considering the total width of all columns in the query and the number of rows for the chunk. Combining this with the concurrency of the executor, you can make a rough estimation of the total memory required for a single query. It is recommended that the total memory for a single chunk does not exceed 16 MiB. ### tidb_isolation_read_engines New in v4.0 > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: SESSION - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): Yes @@ -3240,6 +3253,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified - `start_ts`: The start timestamp of the transaction. - `for_update_ts`: The `for_update_ts` of the previously executed DML statement. This is an internal term of TiDB used for tests. Usually, you can ignore this information. - `error`: The error message, if any. + - `ru_consumption`: Consumed [RU](/tidb-resource-control.md#what-is-request-unit-ru) for executing the statement. ### tidb_last_txn_info New in v4.0.9 @@ -3302,7 +3316,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: No, only applicable to the current TiDB instance that you are connecting to. @@ -3436,7 +3450,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified - Default value: `1024` - Range: `[32, 2147483647]` - Unit: Rows -- This variable is used to set the maximum number of rows in a chunk during the execution process. Setting to too large of a value may cause cache locality issues. +- This variable is used to set the maximum number of rows in a chunk during the execution process. Setting to too large of a value may cause cache locality issues. The recommended value for this variable is no larger than 65536. The number of rows for a chunk directly affects the amount of memory required for a single query. You can roughly estimate the memory needed for a single chunk by considering the total width of all columns in the query and the number of rows for the chunk. Combining this with the concurrency of the executor, you can make a rough estimation of the total memory required for a single query. It is recommended that the total memory for a single chunk does not exceed 16 MiB. When the query involves a large amount of data and a single chunk is insufficient to handle all the data, TiDB processes it multiple times, doubling the chunk size with each processing iteration, starting from [`tidb_init_chunk_size`](#tidb_init_chunk_size) until the chunk size reaches the value of `tidb_max_chunk_size`. ### tidb_max_delta_schema_count New in v2.1.18 and v3.0.5 @@ -3468,7 +3482,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified - Default value: `-1` - Range: `[-1, 256]` - Unit: Threads -- This variable is used to set the maximum concurrency for TiFlash to execute a request. The default value is `-1`, indicating that this system variable is invalid. When the value is `0`, the maximum number of threads is automatically configured by TiFlash. +- This variable is used to set the maximum concurrency for TiFlash to execute a request. The default value is `-1`, indicating that this system variable is invalid and the maximum concurrency depends on the setting of the TiFlash configuration `profiles.default.max_threads`. When the value is `0`, the maximum number of threads is automatically configured by TiFlash. ### tidb_mem_oom_action New in v6.1.0 @@ -3649,7 +3663,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Default value: `1` -- This variable specifies the concurrency of merging statistics for a partitioned table when TiDB analyzes the partitioned table. +- This variable controls the concurrency for merging TopN results of partitioned tables. ### tidb_enable_async_merge_global_stats New in v7.5.0 @@ -3902,7 +3916,7 @@ mysql> desc select count(distinct a) from test.t; - Default value: `ON` - This variable is used to control whether the optimizer estimates the number of rows based on column order correlation -### tidb_opt_enable_hash_join New in v7.4.0 +### tidb_opt_enable_hash_join New in v6.5.6, v7.1.2, and v7.4.0 - Scope: SESSION | GLOBAL - Persists to cluster: Yes @@ -3944,7 +3958,7 @@ mysql> desc select count(distinct a) from test.t; - Default value: `OFF` - This variable controls whether the non-recursive [Common Table Expressions (CTE)](/sql-statements/sql-statement-with.md) can be executed on TiFlash MPP. By default, when this variable is disabled, CTE is executed on TiDB, which has a large performance gap compared with enabling this feature. -### tidb_opt_fix_control New in v7.1.0 +### tidb_opt_fix_control New in v6.5.3 and v7.1.0 @@ -3968,7 +3982,7 @@ mysql> desc select count(distinct a) from test.t; - Default value: `""` - This variable is used to control some internal behaviors of the optimizer. - The optimizer's behavior might vary depending on user scenarios or SQL statements. This variable provides a more fine-grained control over the optimizer and helps to prevent performance regression after upgrading caused by behavior changes in the optimizer. -- For a more detailed introduction, see [Optimizer Fix Controls](https://docs.pingcap.com/tidb/v7.2/optimizer-fix-controls). +- For a more detailed introduction, see [Optimizer Fix Controls](https://docs.pingcap.com/tidb/stable/optimizer-fix-controls/). @@ -4468,13 +4482,13 @@ SHOW WARNINGS; > > - Depending on the specific business scenario, enabling this option might cause a certain degree of throughput reduction (average latency increase) for transactions with frequent lock conflicts. > - This option only takes effect on statements that need to lock a single key. If a statement needs to lock multiple rows at the same time, this option will not take effect on such statements. -> - This feature is introduced in v6.6.0 by the [`tidb_pessimistic_txn_aggressive_locking`](https://docs.pingcap.com/tidb/v6.6/system-variables#tidb_pessimistic_txn_aggressive_locking-new-in-v660) variable, which is disabled by default. +> - This feature is introduced in v6.6.0 by the [`tidb_pessimistic_txn_aggressive_locking`](https://docs-archive.pingcap.com/tidb/v6.6/system-variables#tidb_pessimistic_txn_aggressive_locking-new-in-v660) variable, which is disabled by default. ### tidb_placement_mode New in v6.0.0 > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: SESSION | GLOBAL - Persists to cluster: Yes @@ -4503,8 +4517,8 @@ SHOW WARNINGS; - Scope: SESSION | GLOBAL - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): Yes -- Default value: `2097152` (which is 2 MB) -- Range: `[0, 9223372036854775807]`, in bytes. The memory format with the units "KB|MB|GB|TB" is also supported. `0` means no limit. +- Default value: `2097152` (which is 2 MiB) +- Range: `[0, 9223372036854775807]`, in bytes. The memory format with the units "KiB|MiB|GiB|TiB" is also supported. `0` means no limit. - This variable controls the maximum size of a plan that can be cached in prepared or non-prepared plan cache. If the size of a plan exceeds this value, the plan will not be cached. For more details, see [Memory management of prepared plan cache](/sql-prepared-plan-cache.md#memory-management-of-prepared-plan-cache) and [Non-prepared plan cache](/sql-plan-management.md#usage). ### tidb_pprof_sql_cpu New in v4.0 @@ -4617,7 +4631,7 @@ SHOW WARNINGS; ### tidb_read_consistency New in v5.4.0 - Scope: SESSION -- Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): Yes +- Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): Yes (Note that if [non-transactional DML statements](/non-transactional-dml.md) exist, modifying the value of this variable using hint might not take effect.) - Type: String - Default value: `strict` - This variable is used to control the read consistency for an auto-commit read statement. @@ -4627,7 +4641,7 @@ SHOW WARNINGS; ### tidb_read_staleness New in v5.4.0 - Scope: SESSION -- Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): Yes +- Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Integer - Default value: `0` - Range: `[-2147483648, 0]` @@ -4710,13 +4724,22 @@ SHOW WARNINGS; - For DBaaS providers of TiDB, if a TiDB cluster is a downstream database of another database, to make the TiDB cluster read-only, you might need to use `tidb_restricted_read_only` with [Security Enhanced Mode](#tidb_enable_enhanced_security) enabled, which prevents your customers from using [`tidb_super_read_only`](#tidb_super_read_only-new-in-v531) to make the cluster writable. To achieve this, you need to enable [Security Enhanced Mode](#tidb_enable_enhanced_security), use an admin user with the `SYSTEM_VARIABLES_ADMIN` and `RESTRICTED_VARIABLES_ADMIN` privileges to control `tidb_restricted_read_only`, and let your database users use the root user with the `SUPER` privilege to control [`tidb_super_read_only`](#tidb_super_read_only-new-in-v531) only. - This variable controls the read-only status of the entire cluster. When the variable is `ON`, all TiDB servers in the entire cluster are in the read-only mode. In this case, TiDB only executes the statements that do not modify data, such as `SELECT`, `USE`, and `SHOW`. For other statements such as `INSERT` and `UPDATE`, TiDB rejects executing those statements in the read-only mode. - Enabling the read-only mode using this variable only ensures that the entire cluster finally enters the read-only status. If you have changed the value of this variable in a TiDB cluster but the change has not yet propagated to other TiDB servers, the un-updated TiDB servers are still **not** in the read-only mode. -- When this variable is enabled, the SQL statements being executed are not affected. TiDB only performs the read-only check for the SQL statements **to be** executed. +- TiDB checks the read-only flag before SQL statements are executed. Since v6.2.0, the flag is also checked before SQL statements are committed. This helps prevent the case where long-running [auto commit](/transaction-overview.md#autocommit) statements might modify data after the server has been placed in read-only mode. - When this variable is enabled, TiDB handles the uncommitted transactions in the following ways: - For uncommitted read-only transactions, you can commit the transactions normally. - For uncommitted transactions that are not read-only, SQL statements that perform write operations in these transactions are rejected. - For uncommitted read-only transactions with modified data, the commit of these transactions is rejected. - After the read-only mode is enabled, all users (including the users with the `SUPER` privilege) cannot execute the SQL statements that might write data unless the user is explicitly granted the `RESTRICTED_REPLICA_WRITER_ADMIN` privilege. +### tidb_request_source_type New in v7.4.0 + +- Scope: SESSION +- Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No +- Type: String +- Default value: `""` +- Possible values: `"ddl"`, `"stats"`, `"br"`, `"lightning"`, `"background"` +- This variable is used to explicitly specify the task type for the current session, which is identified and controlled by [Resource Control](/tidb-resource-control.md). For example: `SET @@tidb_request_source_type = "background"`. + ### tidb_retry_limit - Scope: SESSION | GLOBAL @@ -4739,7 +4762,7 @@ SHOW WARNINGS; - Type: Integer - Default value: `2` - Range: `[1, 2]` -- Controls the format version of the newly saved data in the table. In TiDB v4.0, the [new storage row format](https://github.com/pingcap/tidb/blob/master/docs/design/2018-07-19-row-format.md) version `2` is used by default to save new data. +- Controls the format version of the newly saved data in the table. In TiDB v4.0, the [new storage row format](https://github.com/pingcap/tidb/blob/release-7.5/docs/design/2018-07-19-row-format.md) version `2` is used by default to save new data. - If you upgrade from a TiDB version earlier than v4.0.0 to v4.0.0 or later versions, the format version is not changed, and TiDB continues to use the old format of version `1` to write data to the table, which means that **only newly created clusters use the new data format by default**. - Note that modifying this variable does not affect the old data that has been saved, but applies the corresponding version format only to the newly written data after modifying this variable. @@ -4767,7 +4790,7 @@ SHOW WARNINGS; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -4792,7 +4815,7 @@ SHOW WARNINGS; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -4800,8 +4823,8 @@ SHOW WARNINGS; - Default value: `80%` - Range: - You can set the value in the percentage format, which means the percentage of the memory usage relative to the total memory. The value range is `[1%, 99%]`. - - You can also set the value in memory size. The value range is `0` and `[536870912, 9223372036854775807]` in bytes. The memory format with the units "KB|MB|GB|TB" is supported. `0` means no memory limit. - - If this variable is set to a memory size that is less than 512 MB but not `0`, TiDB uses 512 MB as the actual size. + - You can also set the value in memory size. The value range is `0` and `[536870912, 9223372036854775807]` in bytes. The memory format with units "KB|MB|GB|TB" is supported, for example, `90GB` (no space between the number and the unit). `0` means no memory limit. + - If this variable is set to a memory size that is less than 512 MiB but not `0`, TiDB uses 512 MiB as the actual size. - This variable specifies the memory limit for a TiDB instance. When the memory usage of TiDB reaches the limit, TiDB cancels the currently running SQL statement with the highest memory usage. After the SQL statement is successfully canceled, TiDB tries to call Golang GC to immediately reclaim memory to relieve memory stress as soon as possible. - Only the SQL statements with more memory usage than the [`tidb_server_memory_limit_sess_min_size`](/system-variables.md#tidb_server_memory_limit_sess_min_size-new-in-v640) limit are selected as the SQL statements to be canceled first. - Currently, TiDB cancels only one SQL statement at a time. After TiDB completely cancels a SQL statement and recovers resources, if the memory usage is still greater than the limit set by this variable, TiDB starts the next cancel operation. @@ -4810,7 +4833,7 @@ SHOW WARNINGS; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -4823,35 +4846,20 @@ SHOW WARNINGS; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No -- Default value: `134217728` (which is 128 MB) -- Range: `[128, 9223372036854775807]`, in bytes. The memory format with the units "KB|MB|GB|TB" is also supported. +- Default value: `134217728` (which is 128 MiB) +- Range: `[128, 9223372036854775807]`, in bytes. The memory format with units "KB|MB|GB|TB" is supported, for example, `130MB` (no space between the number and the unit). - After you enable the memory limit, TiDB will terminate the SQL statement with the highest memory usage on the current instance. This variable specifies the minimum memory usage of the SQL statement to be terminated. If the memory usage of a TiDB instance that exceeds the limit is caused by too many sessions with low memory usage, you can properly lower the value of this variable to allow more sessions to be canceled. ### tidb_service_scope New in v7.4.0 - - -- Scope: GLOBAL -- Persists to cluster: No -- Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No -- Type: String -- Default value: "" -- Optional Value: "", `background` -- This variable is an instance-level system variable. You can use it to control the service scope of TiDB nodes under the [TiDB distributed execution framework](/tidb-distributed-execution-framework.md). When you set `tidb_service_scope` of a TiDB node to `background`, the TiDB distributed execution framework schedules that TiDB node to execute background tasks, such as [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) and [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md). - > **Note:** > -> - If `tidb_service_scope` is not set for any TiDB node in a cluster, the TiDB distributed execution framework schedules all TiDB nodes to execute background tasks. If you are concerned about performance impact on current business, you can set `tidb_service_scope` to `background` for a few of the TiDB nodes. Only those nodes will execute background tasks. -> - For newly scaled nodes, the TiDB distributed execution framework tasks are not executed by default to avoid consuming the resources of the scaled node. If you want this scaled node to execute background tasks, you need to manually set `tidb_service_scop` of this node to `background`. - - - - +> This TiDB variable is not applicable to TiDB Cloud. - Scope: GLOBAL - Persists to cluster: No @@ -4859,14 +4867,13 @@ SHOW WARNINGS; - Type: String - Default value: "" - Optional Value: "", `background` -- This variable is an instance-level system variable. You can use it to control the service scope of TiDB nodes under the [TiDB distributed execution framework](/tidb-distributed-execution-framework.md). When you set `tidb_service_scope` of a TiDB node to `background`, the TiDB distributed execution framework schedules that TiDB node to execute background tasks, such as [`ADD INDEX`](/sql-statements/sql-statement-add-index.md). +- This variable is an instance-level system variable. You can use it to control the service scope of TiDB nodes under the [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md). When you set `tidb_service_scope` of a TiDB node to `background`, the DXF schedules that TiDB node to execute background tasks, such as [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) and [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md). > **Note:** > -> - If `tidb_service_scope` is not set for any TiDB node in a cluster, the TiDB distributed execution framework schedules all TiDB nodes to execute background tasks. If you are concerned about performance impact on current business, you can set `tidb_service_scope` to `background` for a few of the TiDB nodes. Only those nodes will execute background tasks. -> - For newly scaled nodes, the TiDB distributed execution framework tasks are not executed by default to avoid consuming the resources of the scaled node. If you want this scaled node to execute background tasks, you need to manually set `tidb_service_scop` of this node to `background`. - - +> - If `tidb_service_scope` is not set for any TiDB node in a cluster, the DXF schedules all TiDB nodes to execute background tasks. If you are concerned about performance impact on current business, you can set `tidb_service_scope` to `background` for a few of the TiDB nodes. Only those nodes will execute background tasks. +> - In a cluster with several TiDB nodes, it is strongly recommended to set this system variable to `background` on two or more TiDB nodes. If `tidb_service_scope` is set on a single TiDB node only, when the node is restarted or fails, the task will be rescheduled to other TiDB nodes that lack the `background` setting, which will affect these TiDB nodes. +> - For newly scaled nodes, the DXF tasks are not executed by default to avoid consuming the resources of the scaled node. If you want this scaled node to execute background tasks, you need to manually set `tidb_service_scope` of this node to `background`. ### tidb_session_alias New in v7.4.0 @@ -4901,7 +4908,7 @@ SHOW WARNINGS; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -4946,10 +4953,10 @@ Query OK, 0 rows affected, 1 warning (0.00 sec) - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Boolean - Default value: `ON` -- When accesing a partitioned table in [dynamic pruning mode](/partitioned-table.md#dynamic-pruning-mode), TiDB aggregates the statistics of each partition to generate GlobalStats. This variable controls the generation of GlobalStats when partition statistics are missing. +- When accessing a partitioned table in [dynamic pruning mode](/partitioned-table.md#dynamic-pruning-mode), TiDB aggregates the statistics of each partition to generate GlobalStats. This variable controls the generation of GlobalStats when partition statistics are missing. - If this variable is `ON`, TiDB skips missing partition statistics when generating GlobalStats so the generation of GlobalStats is not affected. - - If this variable is `OFF`, TiDB stops generating GloablStats when it detects any missing partition statistics. + - If this variable is `OFF`, TiDB stops generating GlobalStats when it detects any missing partition statistics. ### tidb_skip_utf8_check @@ -4978,7 +4985,7 @@ Query OK, 0 rows affected, 1 warning (0.00 sec) - Default value: `300` - Range: `[-1, 9223372036854775807]` - Unit: Milliseconds -- This variable is used to output the threshold value of the time consumed by the slow log. When the time consumed by a query is larger than this value, this query is considered as a slow log and its log is output to the slow query log. +- This variable outputs the threshold value of the time consumed by the slow log, and is set to 300 milliseconds by default. When the time consumed by a query is larger than this value, this query is considered as a slow query and its log is output to the slow query log. Note that when the output level of [`log.level`](https://docs.pingcap.com/tidb/v7.5/tidb-configuration-file#level) is `"debug"`, all queries are recorded in the slow query log, regardless of the setting of this variable. ### tidb_slow_query_file @@ -4997,6 +5004,16 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). +### tidb_slow_txn_log_threshold New in v7.0.0 + +- Scope: SESSION +- Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No +- Type: Unsigned integer +- Default value: `0` +- Range: `[0, 9223372036854775807]` +- Unit: Milliseconds +- This variable sets the threshold for slow transaction logging. When the execution time of a transaction exceeds this threshold, TiDB logs detailed information about the transaction. When the value is set to `0`, this feature is disabled. + ### tidb_snapshot - Scope: SESSION @@ -5031,6 +5048,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Integer +- Unit: Byte - Default value: `0`, which means that the memory quota is automatically set to half of the total memory size of the TiDB instance. - Range: `[0, 1099511627776]` - This variable sets the memory quota for the TiDB statistics cache. @@ -5048,7 +5066,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: SESSION | GLOBAL - Persists to cluster: Yes @@ -5195,7 +5213,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -5209,7 +5227,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -5222,7 +5240,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -5230,13 +5248,25 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). - Type: Integer - Default value: `4096` - Range: `[0, 2147483647]` +- Unit: Bytes + + + +- This variable is used to control the length of the SQL string in [statement summary tables](/statement-summary-tables.md) and the [TiDB Dashboard](/dashboard/dashboard-intro.md). + + + + + - This variable is used to control the length of the SQL string in [statement summary tables](/statement-summary-tables.md). + + ### tidb_stmt_summary_max_stmt_count New in v4.0 > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -5250,7 +5280,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -5323,7 +5353,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -5426,7 +5456,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -5451,7 +5481,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -5464,7 +5494,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -5477,7 +5507,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -5490,7 +5520,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -5503,7 +5533,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -5516,7 +5546,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -5529,7 +5559,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No @@ -5542,7 +5572,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No @@ -5555,7 +5585,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: GLOBAL - Persists to cluster: Yes @@ -5599,7 +5629,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). -- This variable is used to control the batch size of transaction commit requests that TiDB sends to TiKV. If most of the transactions in the application workload have a large number of write operations, adjusting this variable to a larger value can improve the performance of batch processing. However, if this variable is set to too large a value and exceeds the limit of TiKV's maximum size of a single log (which is 8 MB by default), the commits might fail. +- This variable is used to control the batch size of transaction commit requests that TiDB sends to TiKV. If most of the transactions in the application workload have a large number of write operations, adjusting this variable to a larger value can improve the performance of batch processing. However, if this variable is set to too large a value and exceeds the limit of TiKV's maximum size of a single log (which is 8 MiB by default), the commits might fail. @@ -5607,7 +5637,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: SESSION | GLOBAL - Persists to cluster: Yes @@ -5632,7 +5662,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: SESSION - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No @@ -5647,7 +5677,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: SESSION - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No @@ -5701,7 +5731,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). - When the window function is pushed down to TiFlash for execution, you can use this variable to control the concurrency level of the window function execution. The possible values are as follows: * -1: the Fine Grained Shuffle feature is disabled. The window function pushed down to TiFlash is executed in a single thread. - * 0: the Fine Grained Shuffle feature is enabled. If [`tidb_max_tiflash_threads`](/system-variables.md#tidb_max_tiflash_threads-new-in-v610) is set to a valid value (greater than 0), then `tiflash_fine_grained_shuffle_stream_count` is set to the value of [`tidb_max_tiflash_threads`](/system-variables.md#tidb_max_tiflash_threads-new-in-v610). Otherwise, it is set to 8. The actual concurrency level of the window function on TiFlash is: min(`tiflash_fine_grained_shuffle_stream_count`, the number of physical threads on TiFlash nodes). + * 0: the Fine Grained Shuffle feature is enabled. If [`tidb_max_tiflash_threads`](/system-variables.md#tidb_max_tiflash_threads-new-in-v610) is set to a valid value (greater than 0), then `tiflash_fine_grained_shuffle_stream_count` is set to the value of [`tidb_max_tiflash_threads`](/system-variables.md#tidb_max_tiflash_threads-new-in-v610). Otherwise, it is automatically estimated based on the CPU resources of the TiFlash compute node. The actual concurrency level of the window function on TiFlash is: min(`tiflash_fine_grained_shuffle_stream_count`, the number of physical threads on TiFlash nodes). * Integer greater than 0: the Fine Grained Shuffle feature is enabled. The window function pushed down to TiFlash is executed in multiple threads. The concurrency level is: min(`tiflash_fine_grained_shuffle_stream_count`, the number of physical threads on TiFlash nodes). - Theoretically, the performance of the window function increases linearly with this value. However, if the value exceeds the actual number of physical threads, it instead leads to performance degradation. @@ -5736,7 +5766,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > > This TiDB variable is not applicable to TiDB Cloud. -- Scope: SESSION | GLOBAL +- Scope: GLOBAL - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Enumeration @@ -5827,7 +5857,7 @@ Internally, the TiDB parser transforms the `SET TRANSACTION ISOLATION LEVEL [REA > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: SESSION - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No @@ -5862,7 +5892,7 @@ Internally, the TiDB parser transforms the `SET TRANSACTION ISOLATION LEVEL [REA > **Note:** > -> This variable is always enabled for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is always enabled for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential). - Scope: GLOBAL - Persists to cluster: Yes @@ -5878,7 +5908,7 @@ Internally, the TiDB parser transforms the `SET TRANSACTION ISOLATION LEVEL [REA - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Integer - Default value: `8` -- Range: `[0, 2147483647]` for TiDB Self-Hosted and [TiDB Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-dedicated), `[8, 2147483647]` for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) +- Range: `[0, 2147483647]` for TiDB Self-Managed and [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated), `[8, 2147483647]` for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) - This variable is a check item in the password complexity check. It checks whether the password length is sufficient. By default, the minimum password length is `8`. This variable takes effect only when [`validate_password.enable`](#validate_passwordenable-new-in-v650) is enabled. - The value of this variable must not be smaller than the expression: `validate_password.number_count + validate_password.special_char_count + (2 * validate_password.mixed_case_count)`. - If you change the value of `validate_password.number_count`, `validate_password.special_char_count`, or `validate_password.mixed_case_count` such that the expression value is larger than `validate_password.length`, the value of `validate_password.length` is automatically changed to match the expression value. @@ -5890,7 +5920,7 @@ Internally, the TiDB parser transforms the `SET TRANSACTION ISOLATION LEVEL [REA - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Integer - Default value: `1` -- Range: `[0, 2147483647]` for TiDB Self-Hosted and [TiDB Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-dedicated), `[1, 2147483647]` for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) +- Range: `[0, 2147483647]` for TiDB Self-Managed and [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated), `[1, 2147483647]` for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) - This variable is a check item in the password complexity check. It checks whether the password contains sufficient uppercase and lowercase letters. This variable takes effect only when [`validate_password.enable`](#validate_passwordenable-new-in-v650) is enabled and [`validate_password.policy`](#validate_passwordpolicy-new-in-v650) is set to `1` (MEDIUM) or larger. - Neither the number of uppercase letters nor the number of lowercase letters in the password can be fewer than the value of `validate_password.mixed_case_count`. For example, when the variable is set to `1`, the password must contain at least one uppercase letter and one lowercase letter. @@ -5901,7 +5931,7 @@ Internally, the TiDB parser transforms the `SET TRANSACTION ISOLATION LEVEL [REA - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Integer - Default value: `1` -- Range: `[0, 2147483647]` for TiDB Self-Hosted and [TiDB Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-dedicated), `[1, 2147483647]` for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) +- Range: `[0, 2147483647]` for TiDB Self-Managed and [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated), `[1, 2147483647]` for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) - This variable is a check item in the password complexity check. It checks whether the password contains sufficient numbers. This variable takes effect only when [`validate_password.enable`](#password_reuse_interval-new-in-v650) is enabled and [`validate_password.policy`](#validate_passwordpolicy-new-in-v650) is set to `1` (MEDIUM) or larger. ### validate_password.policy New in v6.5.0 @@ -5911,7 +5941,7 @@ Internally, the TiDB parser transforms the `SET TRANSACTION ISOLATION LEVEL [REA - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Enumeration - Default value: `1` -- Value options: `0`, `1`, and `2` for TiDB Self-Hosted and [TiDB Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-dedicated); `1` and `2` for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) +- Value options: `0`, `1`, and `2` for TiDB Self-Managed and [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated); `1` and `2` for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) - This variable controls the policy for the password complexity check. This variable takes effect only when [`validate_password.enable`](#password_reuse_interval-new-in-v650) is enabled. The value of this variable determines whether other `validate-password` variables take effect in the password complexity check, except for `validate_password.check_user_name`. - This value of this variable can be `0`, `1`, or `2` (corresponds to LOW, MEDIUM, or STRONG). Different policy levels have different checks: - 0 or LOW: password length. @@ -5925,7 +5955,7 @@ Internally, the TiDB parser transforms the `SET TRANSACTION ISOLATION LEVEL [REA - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Integer - Default value: `1` -- Range: `[0, 2147483647]` for TiDB Self-Hosted and [TiDB Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-dedicated), `[1, 2147483647]` for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) +- Range: `[0, 2147483647]` for TiDB Self-Managed and [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated), `[1, 2147483647]` for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) - This variable is a check item in the password complexity check. It checks whether the password contains sufficient special characters. This variable takes effect only when [`validate_password.enable`](#password_reuse_interval-new-in-v650) is enabled and [`validate_password.policy`](#validate_passwordpolicy-new-in-v650) is set to `1` (MEDIUM) or larger. ### version @@ -5933,14 +5963,14 @@ Internally, the TiDB parser transforms the `SET TRANSACTION ISOLATION LEVEL [REA - Scope: NONE - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Default value: `8.0.11-TiDB-`(tidb version) -- This variable returns the MySQL version, followed by the TiDB version. For example '8.0.11-TiDB-v7.4.0'. +- This variable returns the MySQL version, followed by the TiDB version. For example '8.0.11-TiDB-v{{{ .tidb-version }}}'. ### version_comment - Scope: NONE - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Default value: (string) -- This variable returns additional details about the TiDB version. For example, 'TiDB Server (Apache License 2.0) Community Edition, MySQL 5.7 compatible'. +- This variable returns additional details about the TiDB version. For example, 'TiDB Server (Apache License 2.0) Community Edition, MySQL 8.0 compatible'. ### version_compile_machine @@ -5960,7 +5990,7 @@ Internally, the TiDB parser transforms the `SET TRANSACTION ISOLATION LEVEL [REA > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - Scope: SESSION | GLOBAL - Persists to cluster: Yes diff --git a/table-filter.md b/table-filter.md index a6768fcd76a92..aabbb6b874de0 100644 --- a/table-filter.md +++ b/table-filter.md @@ -1,7 +1,6 @@ --- title: Table Filter summary: Usage of table filter feature in TiDB tools. -aliases: ['/docs/dev/tidb-lightning/tidb-lightning-table-filter/','/docs/dev/reference/tools/tidb-lightning/table-filter/','/tidb/dev/tidb-lightning-table-filter/'] --- # Table Filter diff --git a/telemetry.md b/telemetry.md index 6c1cb5bf87112..5dcd2be951c14 100644 --- a/telemetry.md +++ b/telemetry.md @@ -1,7 +1,6 @@ --- title: Telemetry summary: Learn the telemetry feature, how to disable the feature and view its status. -aliases: ['/docs/dev/telemetry/'] --- # Telemetry @@ -15,7 +14,7 @@ When the telemetry is enabled, TiDB, TiUP and TiDB Dashboard collect usage infor ## What is shared? -The following sections describe the shared usage information in detail for each component. The usage details that get shared might change over time. These changes (if any) will be announced in [release notes](/releases/release-notes.md). +The following sections describe the shared usage information in detail for each component. The usage details that get shared might change over time. These changes (if any) will be announced in [release notes](https://docs.pingcap.com/releases/tidb-self-managed/). > **Note:** > @@ -286,4 +285,4 @@ To meet compliance requirements in different countries or regions, the usage inf - For IP addresses from the Chinese mainland, usage information is sent to and stored on cloud servers in the Chinese mainland. - For IP addresses from outside of the Chinese mainland, usage information is sent to and stored on cloud servers in the US. -See [PingCAP Privacy Policy](https://en.pingcap.com/privacy-policy/) for details. +See [PingCAP Privacy Policy](https://www.pingcap.com/privacy-policy/) for details. diff --git a/templates/copyright.tex b/templates/copyright.tex index 93eb006563050..180d0d9e03f73 100644 --- a/templates/copyright.tex +++ b/templates/copyright.tex @@ -1,4 +1,4 @@ \noindent \rule{\textwidth}{1pt} -© 2023 PingCAP. All Rights Reserved. \ No newline at end of file +© 2026 PingCAP. All Rights Reserved. diff --git a/templates/deeplist.tex b/templates/deeplist.tex new file mode 100644 index 0000000000000..3776db45b8f3a --- /dev/null +++ b/templates/deeplist.tex @@ -0,0 +1,24 @@ +\usepackage{enumitem} +\setlistdepth{9} + +\setlist[itemize,1]{label=$\bullet$} +\setlist[itemize,2]{label=$\bullet$} +\setlist[itemize,3]{label=$\bullet$} +\setlist[itemize,4]{label=$\bullet$} +\setlist[itemize,5]{label=$\bullet$} +\setlist[itemize,6]{label=$\bullet$} +\setlist[itemize,7]{label=$\bullet$} +\setlist[itemize,8]{label=$\bullet$} +\setlist[itemize,9]{label=$\bullet$} +\renewlist{itemize}{itemize}{9} + +\setlist[enumerate,1]{label=$\arabic*.$} +\setlist[enumerate,2]{label=$\alph*.$} +\setlist[enumerate,3]{label=$\roman*.$} +\setlist[enumerate,4]{label=$\arabic*.$} +\setlist[enumerate,5]{label=$\alpha*$} +\setlist[enumerate,6]{label=$\roman*.$} +\setlist[enumerate,7]{label=$\arabic*.$} +\setlist[enumerate,8]{label=$\alph*.$} +\setlist[enumerate,9]{label=$\roman*.$} +\renewlist{enumerate}{enumerate}{9} diff --git a/three-data-centers-in-two-cities-deployment.md b/three-data-centers-in-two-cities-deployment.md index 27352220f4174..1b08e5d76df41 100644 --- a/three-data-centers-in-two-cities-deployment.md +++ b/three-data-centers-in-two-cities-deployment.md @@ -1,7 +1,6 @@ --- title: Three Availability Zones in Two Regions Deployment summary: Learn the deployment solution to three availability zones in two regions. -aliases: ['/docs/dev/three-data-centers-in-two-cities-deployment/'] --- # Three Availability Zones in Two Regions Deployment @@ -114,8 +113,8 @@ tikv_servers: - host: 10.63.10.34 config: server.labels: { az: "3", replication zone: "5", rack: "5", host: "34" } - raftstore.raft-min-election-timeout-ticks: 1000 - raftstore.raft-max-election-timeout-ticks: 1200 + raftstore.raft-min-election-timeout-ticks: 50 + raftstore.raft-max-election-timeout-ticks: 60 monitoring_servers: - host: 10.63.10.60 @@ -175,11 +174,15 @@ In the deployment of three AZs in two regions, to optimize performance, you need - Optimize the network configuration of the TiKV node in another region (San Francisco). Modify the following TiKV parameters for AZ3 in San Francisco and try to prevent the replica in this TiKV node from participating in the Raft election. ```yaml - raftstore.raft-min-election-timeout-ticks: 1000 - raftstore.raft-max-election-timeout-ticks: 1200 + raftstore.raft-min-election-timeout-ticks: 50 + raftstore.raft-max-election-timeout-ticks: 60 ``` -- Configure scheduling. After the cluster is enabled, use the `tiup ctl:v pd` tool to modify the scheduling policy. Modify the number of TiKV Raft replicas. Configure this number as planned. In this example, the number of replicas is five. +> **Note:** +> +> Using `raftstore.raft-min-election-timeout-ticks` and `raftstore.raft-max-election-timeout-ticks` to configure larger election timeout ticks for a TiKV node can significantly decrease the likelihood of Regions on that node becoming Leaders. However, in a disaster scenario where some TiKV nodes are offline and the remaining active TiKV nodes lag behind in Raft logs, only Regions on this TiKV node with large election timeout ticks can become Leaders. Because Regions on this TiKV node must wait for at least the duration set by `raftstore.raft-min-election-timeout-ticks' before initiating an election, it is recommended to avoid setting these values excessively large to prevent potential impact on the cluster availability in such scenarios. + +- Configure scheduling. After the cluster is enabled, use the `tiup ctl:v{CLUSTER_VERSION} pd` tool to modify the scheduling policy. Modify the number of TiKV Raft replicas. Configure this number as planned. In this example, the number of replicas is five. ```bash config set max-replicas 5 diff --git a/ticdc-deployment-topology.md b/ticdc-deployment-topology.md index 75a724821e15c..c618076648458 100644 --- a/ticdc-deployment-topology.md +++ b/ticdc-deployment-topology.md @@ -1,7 +1,6 @@ --- title: TiCDC Deployment Topology summary: Learn the deployment topology of TiCDC based on the minimal TiDB topology. -aliases: ['/docs/dev/ticdc-deployment-topology/'] --- # TiCDC Deployment Topology @@ -24,6 +23,10 @@ TiCDC is a tool for replicating the incremental data of TiDB, introduced in TiDB | CDC | 3 | 8 VCore 16GB * 1 | 10.0.1.11
    10.0.1.12
    10.0.1.13 | Default port
    Global directory configuration | | Monitoring & Grafana | 1 | 4 VCore 8GB * 1 500GB (ssd) | 10.0.1.11 | Default port
    Global directory configuration | +> **Note:** +> +> The IP addresses of the instances are given as examples only. In your actual deployment, replace the IP addresses with your actual IP addresses. + ### Topology templates - [The simple template for the TiCDC topology](https://github.com/pingcap/docs/blob/master/config-templates/simple-cdc.yaml) diff --git a/ticdc/deploy-ticdc.md b/ticdc/deploy-ticdc.md index 37a989a5ef834..f2479813337ec 100644 --- a/ticdc/deploy-ticdc.md +++ b/ticdc/deploy-ticdc.md @@ -95,7 +95,7 @@ tiup cluster upgrade --transfer-timeout 600 > **Note:** > -> In the preceding command, you need to replace `` and `` with the actual cluster name and cluster version. For example, the version can be v7.4.0. +> In the preceding command, you need to replace `` and `` with the actual cluster name and cluster version. For example, the version can be v{{{ .tidb-version }}}. ### Upgrade cautions @@ -152,10 +152,10 @@ See [Enable TLS Between TiDB Components](/enable-tls-between-components.md). ## View TiCDC status using the command-line tool -Run the following command to view the TiCDC cluster status. Note that you need to replace `v` with the TiCDC cluster version, such as `v7.4.0`: +Run the following command to view the TiCDC cluster status. Note that you need to replace `v` with the TiCDC cluster version, such as `v{{{ .tidb-version }}}`: ```shell -tiup ctl:v cdc capture list --server=http://10.0.10.25:8300 +tiup cdc:v cli capture list --server=http://10.0.10.25:8300 ``` ```shell diff --git a/ticdc/integrate-confluent-using-ticdc.md b/ticdc/integrate-confluent-using-ticdc.md index 43b71808f7492..1cfe340f376bb 100644 --- a/ticdc/integrate-confluent-using-ticdc.md +++ b/ticdc/integrate-confluent-using-ticdc.md @@ -99,7 +99,7 @@ The preceding steps are performed in a lab environment. You can also deploy a cl 2. Create a changefeed to replicate incremental data to Confluent Cloud: ```shell - tiup ctl:v cdc changefeed create --server="http://127.0.0.1:8300" --sink-uri="kafka:///ticdc-meta?protocol=avro&replication-factor=3&enable-tls=true&auto-create-topic=true&sasl-mechanism=plain&sasl-user=&sasl-password=" --schema-registry="https://:@" --changefeed-id="confluent-changefeed" --config changefeed.conf + tiup cdc:v cli changefeed create --server="http://127.0.0.1:8300" --sink-uri="kafka:///ticdc-meta?protocol=avro&replication-factor=3&enable-tls=true&auto-create-topic=true&sasl-mechanism=plain&sasl-user=&sasl-password=" --schema-registry="https://:@" --changefeed-id="confluent-changefeed" --config changefeed.conf ``` You need to replace the values of the following fields with those created or recorded in [Step 2. Create an access key pair](#step-2-create-an-access-key-pair): @@ -114,7 +114,7 @@ The preceding steps are performed in a lab environment. You can also deploy a cl Note that you should encode `` based on [HTML URL Encoding Reference](https://www.w3schools.com/tags/ref_urlencode.asp) before replacing its value. After you replace all the preceding fields, the configuration file is as follows: ```shell - tiup ctl:v cdc changefeed create --server="http://127.0.0.1:8300" --sink-uri="kafka://xxx-xxxxx.ap-east-1.aws.confluent.cloud:9092/ticdc-meta?protocol=avro&replication-factor=3&enable-tls=true&auto-create-topic=true&sasl-mechanism=plain&sasl-user=L5WWA4GK4NAT2EQV&sasl-password=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" --schema-registry="https://7NBH2CAFM2LMGTH7:xxxxxxxxxxxxxxxxxx@yyy-yyyyy.us-east-2.aws.confluent.cloud" --changefeed-id="confluent-changefeed" --config changefeed.conf + tiup cdc:v cli changefeed create --server="http://127.0.0.1:8300" --sink-uri="kafka://xxx-xxxxx.ap-east-1.aws.confluent.cloud:9092/ticdc-meta?protocol=avro&replication-factor=3&enable-tls=true&auto-create-topic=true&sasl-mechanism=plain&sasl-user=L5WWA4GK4NAT2EQV&sasl-password=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" --schema-registry="https://7NBH2CAFM2LMGTH7:xxxxxxxxxxxxxxxxxx@yyy-yyyyy.us-east-2.aws.confluent.cloud" --changefeed-id="confluent-changefeed" --config changefeed.conf ``` - Run the command to create a changefeed. @@ -132,7 +132,7 @@ The preceding steps are performed in a lab environment. You can also deploy a cl 3. After creating the changefeed, run the following command to check the changefeed status: ```shell - tiup ctl:v cdc changefeed list --server="http://127.0.0.1:8300" + tiup cdc:v cli changefeed list --server="http://127.0.0.1:8300" ``` You can refer to [Manage TiCDC Changefeeds](/ticdc/ticdc-manage-changefeed.md) to manage the changefeed. diff --git a/ticdc/monitor-ticdc.md b/ticdc/monitor-ticdc.md index 66feb10fcd071..08f355e63f8bf 100644 --- a/ticdc/monitor-ticdc.md +++ b/ticdc/monitor-ticdc.md @@ -97,8 +97,8 @@ The description of each metric in the **Events** panel is as follows: - Entry sorter sort duration percentile: The time (P95, P99, and P999) spent by TiCDC sorting events within one second - Entry sorter merge duration: The histogram of the time spent by TiCDC nodes merging sorted events - Entry sorter merge duration percentile: The time (P95, P99, and P999) spent by TiCDC merging sorted events within one second -- Mounter unmarshal duration: The histogram of the time spent by TiCDC nodes unmarshaling events -- Mounter unmarshal duration percentile: The time (P95, P99, and P999) spent by TiCDC unmarshaling events within one second +- Mounter unmarshal duration: The histogram of the time spent by TiCDC nodes unmarshalling events +- Mounter unmarshal duration percentile: The time (P95, P99, and P999) spent by TiCDC unmarshalling events within one second - KV client dispatch events/s: The number of events that the KV client module dispatches among the TiCDC nodes - KV client batch resolved size: The batch size of resolved timestamp messages that TiKV sends to TiCDC diff --git a/ticdc/ticdc-alert-rules.md b/ticdc/ticdc-alert-rules.md index 7e245b53e4c61..855819fcc8ae9 100644 --- a/ticdc/ticdc-alert-rules.md +++ b/ticdc/ticdc-alert-rules.md @@ -16,7 +16,7 @@ For critical alerts, you need to pay close attention to abnormal monitoring metr - Alert rule: - (time() - ticdc_owner_checkpoint_ts / 1000) > 600 + `ticdc_owner_checkpoint_ts_lag > 600` - Description: @@ -24,13 +24,13 @@ For critical alerts, you need to pay close attention to abnormal monitoring metr - Solution: - See [TiCDC Handle Replication Interruption](/ticdc/troubleshoot-ticdc.md#how-do-i-handle-replication-interruptions). + See [TiCDC Handles Replication Interruption](/ticdc/troubleshoot-ticdc.md#how-do-i-handle-replication-interruptions). -## `cdc_resolvedts_high_delay` +### `cdc_resolvedts_high_delay` - Alert rule: - (time() - ticdc_owner_resolved_ts / 1000) > 300 + `ticdc_owner_resolved_ts_lag > 300` - Description: @@ -38,7 +38,21 @@ For critical alerts, you need to pay close attention to abnormal monitoring metr - Solution: - See [TiCDC Handle Replication Interruption](/ticdc/troubleshoot-ticdc.md#how-do-i-handle-replication-interruptions). + See [TiCDC Handles Replication Interruption](/ticdc/troubleshoot-ticdc.md#how-do-i-handle-replication-interruptions). + +### `ticdc_changefeed_failed` + +- Alert rule: + + `(max_over_time(ticdc_owner_status[1m]) == 2) > 0` + +- Description: + + A replication task encounters an unrecoverable error and enters the failed state. + +- Solution: + + This alert is similar to replication interruption. See [TiCDC Handles Replication Interruption](/ticdc/troubleshoot-ticdc.md#how-do-i-handle-replication-interruptions). ### `ticdc_processor_exit_with_error_count` @@ -52,7 +66,7 @@ For critical alerts, you need to pay close attention to abnormal monitoring metr - Solution: - See [TiCDC Handle Replication Interruption](/ticdc/troubleshoot-ticdc.md#how-do-i-handle-replication-interruptions). + See [TiCDC Handles Replication Interruption](/ticdc/troubleshoot-ticdc.md#how-do-i-handle-replication-interruptions). ## Warning alerts @@ -98,7 +112,7 @@ Warning alerts are a reminder for an issue or error. - Solution: - See [TiCDC Handle Replication Interruption](/ticdc/troubleshoot-ticdc.md#how-do-i-handle-replication-interruptions). + See [TiCDC Handles Replication Interruption](/ticdc/troubleshoot-ticdc.md#how-do-i-handle-replication-interruptions). ### `ticdc_puller_entry_sorter_sort_bucket` @@ -132,7 +146,7 @@ Warning alerts are a reminder for an issue or error. - Alert rule: - `changes(tikv_cdc_min_resolved_ts[1m]) < 1 and ON (instance) tikv_cdc_region_resolve_status{status="resolved"} > 0` + `changes(tikv_cdc_min_resolved_ts[1m]) < 1 and ON (instance) tikv_cdc_region_resolve_status{status="resolved"} > 0 and ON (instance) tikv_cdc_captured_region_total > 0` - Description: @@ -182,4 +196,4 @@ Warning alerts are a reminder for an issue or error. - Solution: - Collect TiCDC logs to locate the root cause. \ No newline at end of file + Collect TiCDC logs to locate the root cause. diff --git a/ticdc/ticdc-avro-checksum-verification.md b/ticdc/ticdc-avro-checksum-verification.md index 1cc9e89a290c8..ae7aa54bb2f3c 100644 --- a/ticdc/ticdc-avro-checksum-verification.md +++ b/ticdc/ticdc-avro-checksum-verification.md @@ -7,7 +7,7 @@ summary: Introduce the detailed implementation of TiCDC row data checksum verifi This document introduces how to consume data sent to Kafka by TiCDC and encoded by Avro protocol using Golang, and how to perform data verification using the [Single-row data checksum feature](/ticdc/ticdc-integrity-check.md). -The source code of this example is available in the [`avro-checksum-verification`](https://github.com/pingcap/tiflow/tree/master/examples/golang/avro-checksum-verification) directory. +The source code of this example is available in the [`avro-checksum-verification`](https://github.com/pingcap/tiflow/tree/release-7.5/examples/golang/avro-checksum-verification) directory. The example in this document uses [kafka-go](https://github.com/segmentio/kafka-go) to create a simple Kafka consumer program. This program continuously reads data from a specified topic, calculates the checksum, and verifies its value. diff --git a/ticdc/ticdc-avro-protocol.md b/ticdc/ticdc-avro-protocol.md index b616631729b6c..3e4218358d8da 100644 --- a/ticdc/ticdc-avro-protocol.md +++ b/ticdc/ticdc-avro-protocol.md @@ -95,6 +95,16 @@ The `fields` in the key contains only primary key columns or unique index column The data format of Value is the same as that of Key, by default. However, `fields` in the Value contains all columns, not just the primary key columns. +> **Note:** +> +> The Avro protocol encodes DML events as follows: +> +> - For Delete events, Avro only encodes the Key part. The Value part is empty. +> - For Insert events, Avro encodes all column data to the Value part. +> - For Update events, Avro encodes only all column data that is updated to the Value part. +> +> The Avro protocol does not encode the old values for Update and Delete events. Additionally, to be compatible with most Confluent sink connectors that rely on `null` records to identify deletions (`delete.on.null`), Delete events do not include extension information, such as `_tidb_commit_ts`, even when `enable-tidb-extension` is enabled. If you need these features, consider using other protocols such as Canal-JSON or Debezium. + After you enable [`enable-tidb-extension`](#tidb-extension-fields), the data format of the Value will be as follows: ``` diff --git a/ticdc/ticdc-canal-json.md b/ticdc/ticdc-canal-json.md index d01b197f96852..f7c4807d3b13a 100644 --- a/ticdc/ticdc-canal-json.md +++ b/ticdc/ticdc-canal-json.md @@ -42,7 +42,7 @@ cdc cli changefeed create --server=http://127.0.0.1:8300 --changefeed-id="kafka- ## Definitions of message formats -This section describes the formats of DDL Event, DML Event and WATERMARK Event, and how the data is resolved on the consumer side. +This section describes the formats of DDL Event, DML Event and WATERMARK Event, and how the data is parsed on the consumer side. ### DDL Event @@ -64,7 +64,7 @@ TiCDC encodes a DDL Event into the following Canal-JSON format. "data": null, "old": null, "_tidb": { // TiDB extension field - "commitTs": 163963309467037594 + "commitTs": 429918007904436226 // A TiDB TSO timestamp } } ``` @@ -133,7 +133,7 @@ TiCDC encodes a row of DML data change event as follows: ], "old": null, "_tidb": { // TiDB extension field - "commitTs": 163963314122145239 + "commitTs": 429918007904436226 // A TiDB TSO timestamp } } ``` @@ -162,14 +162,14 @@ The following is an example of the WATERMARK Event. "data": null, "old": null, "_tidb": { // TiDB extension field - "watermarkTs": 429918007904436226 + "watermarkTs": 429918007904436226 // A TiDB TSO timestamp } } ``` -### Data resolution on the consumer side +### Data parsing on the consumer side -As you can see from the example above, Canal-JSON has a uniform data format, with different field filling rules for different Event types. You can use a uniform method to resolve this JSON format data, and then determine the Event type by checking the field values. +As you can see from the example above, Canal-JSON has a unified data format, with different field filling rules for different Event types. Consumers can parse data in this JSON format using a unified method, and then determine the Event type by checking the field values: * When `isDdl` is `true`, the message contains a DDL Event. * When `isDdl` is `false`, you need to further check the `type` field. If `type` is `TIDB_WATERMARK`, it is a WATERMARK Event; otherwise, it is a DML Event. diff --git a/ticdc/ticdc-changefeed-config.md b/ticdc/ticdc-changefeed-config.md index 9c9e26d607989..b980a37a8aa78 100644 --- a/ticdc/ticdc-changefeed-config.md +++ b/ticdc/ticdc-changefeed-config.md @@ -16,7 +16,7 @@ cdc cli changefeed create --server=http://10.0.10.25:8300 --sink-uri="mysql://ro ```shell Create changefeed successfully! ID: simple-replication-task -Info: {"upstream_id":7178706266519722477,"namespace":"default","id":"simple-replication-task","sink_uri":"mysql://root:xxxxx@127.0.0.1:4000/?time-zone=","create_time":"2022-12-19T15:05:46.679218+08:00","start_ts":438156275634929669,"engine":"unified","config":{"case_sensitive":true,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":3600000000000,"filter":{"rules":["test.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"none","enable_partition_separator":false},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"storage":""}},"state":"normal","creator_version":"v6.5.0"} +Info: {"upstream_id":7178706266519722477,"namespace":"default","id":"simple-replication-task","sink_uri":"mysql://root:xxxxx@127.0.0.1:4000/?time-zone=","create_time":"{{{ .tidb-release-date }}}T15:05:46.679218+08:00","start_ts":438156275634929669,"engine":"unified","config":{"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":3600000000000,"filter":{"rules":["test.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"none","enable_partition_separator":false},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"storage":""}},"state":"normal","creator_version":"v{{{ .tidb-version }}}"} ``` - `--changefeed-id`: The ID of the replication task. The format must match the `^[a-zA-Z0-9]+(\-[a-zA-Z0-9]+)*$` regular expression. If this ID is not specified, TiCDC automatically generates a UUID (the version 4 format) as the ID. @@ -26,7 +26,7 @@ Info: {"upstream_id":7178706266519722477,"namespace":"default","id":"simple-repl [scheme]://[userinfo@][host]:[port][/path]?[query_parameters] ``` - When the sink URI contains special characters such as `! * ' ( ) ; : @ & = + $ , / ? % # [ ]`, you need to escape the special characters, for example, in [URI Encoder](https://meyerweb.com/eric/tools/dencoder/). + When the sink URI contains special characters such as `! * ' ( ) ; : @ & = + $ , / ? % # [ ]`, you need to escape the special characters, for example, in [URI Encoder](https://www.urlencoder.org/). - `--start-ts`: Specifies the starting TSO of the changefeed. From this TSO, the TiCDC cluster starts pulling data. The default value is the current time. - `--target-ts`: Specifies the ending TSO of the changefeed. To this TSO, the TiCDC cluster stops pulling data. The default value is empty, which means that TiCDC does not automatically stop pulling data. @@ -43,9 +43,9 @@ This section introduces the configuration of a replication task. # memory-quota = 1073741824 # Specifies whether the database names and tables in the configuration file are case-sensitive. -# The default value is true. +# Starting from v7.5.0, the default value changes from true to false. # This configuration item affects configurations related to filter and sink. -case-sensitive = true +case-sensitive = false # Specifies whether to enable the Syncpoint feature, which is supported since v6.3.0 and is disabled by default. # Since v6.4.0, only the changefeed with the SYSTEM_VARIABLES_ADMIN or SUPER privilege can use the TiCDC Syncpoint feature. @@ -64,10 +64,20 @@ case-sensitive = true # Note: This configuration item only takes effect if the downstream is TiDB. # sync-point-retention = "1h" -# Specifies the SQL mode used when parsing DDL statements. Multiple modes are separated by commas. +# Starting from v6.5.6, v7.1.3, and v7.5.0, this configuration item specifies the SQL mode used when parsing DDL statements. Multiple modes are separated by commas. # The default value is the same as the default SQL mode of TiDB. # sql-mode = "ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION" +# The default value is false, which means that Bi-Directional Replication (BDR) mode is disabled. +# For more information, see https://docs.pingcap.com/tidb/stable/ticdc-bidirectional-replication +# bdr-mode = false + +# The duration for which the changefeed is allowed to automatically retry when internal errors or exceptions occur. The default value is 30 minutes. +# The changefeed enters the failed state if internal errors or exceptions occur in the changefeed and persist longer than the duration set by this parameter. +# When the changefeed is in the failed state, you need to restart the changefeed manually for recovery. +# The format of this parameter is "h m s", for example, "1h30m30s". +changefeed-error-stuck-duration = "30m" + [mounter] # The number of threads with which the mounter decodes KV data. The default value is 16. # worker-num = 16 @@ -94,7 +104,7 @@ rules = ['*.*', '!test.*'] # The second event filter rule. # matcher = ["test.fruit"] # matcher is an allow list, which means this rule only applies to the fruit table in the test database. -# ignore-event = ["drop table", "delete"] # Ignore the `drop table` DDL events and the `delete` DML events. +# ignore-event = ["drop table", "delete"] # Ignore the `drop table` DDL events and the `delete` DML events. Note that when a value in the clustered index column is updated in TiDB, TiCDC splits an `UPDATE` event into `DELETE` and `INSERT` events. TiCDC cannot identify such events as `UPDATE` events and thus cannot correctly filter out such events. # ignore-sql = ["^drop table", "alter table"] # Ignore DDL statements that start with `drop table` or contain `alter table`. # ignore-insert-value-expr = "price > 1000 and origin = 'no where'" # Ignore insert DMLs that contain the conditions "price > 1000" and "origin = 'no where'". @@ -104,8 +114,8 @@ rules = ['*.*', '!test.*'] # The value is "false" by default. Set it to "true" to enable this feature. enable-table-across-nodes = false # When `enable-table-across-nodes` is enabled, there are two allocation modes: -# 1. Allocate tables based on the number of Regions, so that each TiCDC node handles roughly the same number of Regions. If the number of Regions for a table exceeds the value of `region-threshold`, the table will be allocated to multiple nodes for replication. The default value of `region-threshold` is 10000. -# region-threshold = 10000 +# 1. Allocate tables based on the number of Regions, so that each TiCDC node handles roughly the same number of Regions. If the number of Regions for a table exceeds the value of `region-threshold`, the table will be allocated to multiple nodes for replication. The default value of `region-threshold` is 100000. +# region-threshold = 100000 # 2. Allocate tables based on the write traffic, so that each TiCDC node handles roughly the same number of modified rows. Only when the number of modified rows per minute in a table exceeds the value of `write-key-threshold`, will this allocation take effect. # write-key-threshold = 30000 # Note: @@ -122,7 +132,7 @@ enable-table-across-nodes = false # For example, if you specify the routing rule for a matcher as the string `code`, then all Pulsar messages that match that matcher will be routed with `code` as the key. # dispatchers = [ # {matcher = ['test1.*', 'test2.*'], topic = "Topic expression 1", partition = "index-value"}, -# {matcher = ['test3.*', 'test4.*'], topic = "Topic expression 2", partition = "index-value", index-name="index1"}, +# {matcher = ['test3.*', 'test4.*'], topic = "Topic expression 2", partition = "index-value", index = "index1"}, # {matcher = ['test1.*', 'test5.*'], topic = "Topic expression 3", partition = "table"}, # {matcher = ['test6.*'], partition = "columns", columns = "['a', 'b']"} # {matcher = ['test7.*'], partition = "ts"} @@ -161,7 +171,7 @@ delete-only-output-handle-key-columns = false # encoder-concurrency = 32 # Specifies whether to enable kafka-sink-v2 that uses the kafka-go sink library. -# Note: This configuration item only takes effect if the downstream is MQ. +# Note: This configuration item is experimental, and only takes effect if the downstream is MQ. # The default value is false. # enable-kafka-sink-v2 = false @@ -177,7 +187,7 @@ delete-only-output-handle-key-columns = false # Date separator type used in the file directory. Value options are `none`, `year`, `month`, and `day`. `day` is the default value and means separating files by day. For more information, see . # Note: This configuration item only takes effect if the downstream is a storage service. date-separator = 'day' -# Whether to use partitions as the separation string. The default value is true, which means that partitions in a table are stored in separate directories. It is recommended that you keep the value as `true` to avoid potential data loss in downstream partitioned tables . For usage examples, see . +# Whether to use partitions as the separation string. The default value is true, which means that partitions in a table are stored in separate directories. It is recommended that you keep the value as `true` to avoid potential data loss in downstream partitioned tables . For usage examples, see . # Note: This configuration item only takes effect if the downstream is a storage service. enable-partition-separator = true @@ -194,6 +204,10 @@ enable-partition-separator = true # The encoding method of binary data, which can be 'base64' or 'hex'. The default value is 'base64'. # binary-encoding-method = 'base64' +[sink.open] +# Whether to output the value before the row data changes. The default value is true. When it is disabled, the UPDATE event does not output the "p" field. This is introduced in v7.5.2. +# output-old-value = true + # Specifies the replication consistency configurations for a changefeed when using the redo log. For more information, see https://docs.pingcap.com/tidb/stable/ticdc-sink-to-mysql#eventually-consistent-replication-in-disaster-scenarios. # Note: The consistency-related configuration items only take effect when the downstream is a database and the redo log feature is enabled. [consistent] @@ -208,9 +222,21 @@ flush-interval = 2000 # The storage URI of the redo log. # The default value is empty. storage = "" -# Specifies whether to store the redo log in a file. +# Specifies whether to store the redo log in a local file. # The default value is false. use-file-backend = false +# The number of encoding and decoding workers in the redo module. +# The default value is 16. +encoding-worker-num = 16 +# The number of flushing workers in the redo module. +# The default value is 8. +flush-worker-num = 8 +# The behavior to compress redo log files (introduced in v6.5.6, v7.1.3, and v7.5.1). +# Available options are "" and "lz4". The default value is "", which means no compression. +compression = "" +# The concurrency for uploading a single redo file (introduced in v6.5.6, v7.1.3, and v7.5.1). +# The default value is 1, which means concurrency is disabled. +flush-concurrency = 1 [integrity] # Whether to enable the checksum validation for single-row data. The default value is "none", which means to disable the feature. Value options are "none" and "correctness". @@ -235,6 +261,18 @@ sasl-oauth-grant-type = "client_credentials" # The audience in the Kafka SASL OAUTHBEARER authentication. The default value is empty. This parameter is optional when the OAUTHBEARER authentication is used. sasl-oauth-audience = "kafka" +# The following configuration item controls whether to output the original data change event. The default value is false. For more information, see https://docs.pingcap.com/tidb/v7.5/ticdc-split-update-behavior#control-whether-to-split-primary-or-unique-key-update-events. +# output-raw-change-event = false + +# The following configuration is only required when using Avro as the protocol and AWS Glue Schema Registry: +# Please refer to the section "Integrate TiCDC with AWS Glue Schema Registry" in the document "Sync Data to Kafka": https://docs.pingcap.com/tidb/dev/ticdc-sink-to-kafka#integrate-ticdc-with-aws-glue-schema-registry +# [sink.kafka-config.glue-schema-registry-config] +# region="us-west-1" +# registry-name="ticdc-test" +# access-key="xxxx" +# secret-access-key="xxxx" +# token="xxxx" + # The following parameters take effect only when the downstream is Pulsar. [sink.pulsar-config] # Authentication on the Pulsar server is done using a token. Specify the value of the token. @@ -275,4 +313,29 @@ batching-max-messages=1000 batching-max-publish-delay=10 # The timeout for a Pulsar producer to send a message. The value is 30 seconds by default. send-timeout=30 + +# The following configuration item controls whether to output the original data change event. The default value is false. For more information, see https://docs.pingcap.com/tidb/v7.5/ticdc-split-update-behavior#control-whether-to-split-primary-or-unique-key-update-events. +# output-raw-change-event = false + +[sink.cloud-storage-config] +# The concurrency for saving data changes to the downstream cloud storage. +# The default value is 16. +worker-count = 16 +# The interval for saving data changes to the downstream cloud storage. +# The default value is "2s". +flush-interval = "2s" +# A data change file is saved to the cloud storage when the number of bytes in this file exceeds `file-size`. +# The default value is 67108864 (this is, 64 MiB). +file-size = 67108864 +# The duration to retain files, which takes effect only when `date-separator` is configured as `day`. Assume that `file-expiration-days = 1` and `file-cleanup-cron-spec = "0 0 0 * * *"`, then TiCDC performs daily cleanup at 00:00:00 for files saved beyond 24 hours. For example, at 00:00:00 on 2023/12/02, TiCDC cleans up files generated before 2023/12/01, while files generated on 2023/12/01 remain unaffected. +# The default value is 0, which means file cleanup is disabled. +file-expiration-days = 0 +# The running cycle of the scheduled cleanup task, compatible with the crontab configuration, with a format of ` ` +# The default value is "0 0 2 * * *", which means that the cleanup task is executed every day at 2 AM. +file-cleanup-cron-spec = "0 0 2 * * *" +# The concurrency for uploading a single file. +# The default value is 1, which means concurrency is disabled. +flush-concurrency = 1 +# The following configuration item controls whether to output the original data change event. The default value is false. For more information, see https://docs.pingcap.com/tidb/v7.5/ticdc-split-update-behavior#control-whether-to-split-primary-or-unique-key-update-events. +output-raw-change-event = false ``` diff --git a/ticdc/ticdc-changefeed-overview.md b/ticdc/ticdc-changefeed-overview.md index 07e630a34c7d2..f962600c67865 100644 --- a/ticdc/ticdc-changefeed-overview.md +++ b/ticdc/ticdc-changefeed-overview.md @@ -15,15 +15,16 @@ The state of a replication task represents the running status of the replication The states in the preceding state transfer diagram are described as follows: -- `Normal`: The replication task runs normally and the checkpoint-ts proceeds normally. +- `Normal`: The replication task runs normally and the checkpoint-ts proceeds normally. A changefeed in this state blocks GC to advance. - `Stopped`: The replication task is stopped, because the user manually pauses the changefeed. The changefeed in this state blocks GC operations. -- `Warning`: The replication task returns an error. The replication cannot continue due to some recoverable errors. The changefeed in this state keeps trying to resume until the state transfers to `Normal`. The maximum retry time is 30 minutes. If it exceeds this time, the changefeed enters a failed state. The changefeed in this state blocks GC operations. +- `Warning`: The replication task returns an error. The replication cannot continue due to some recoverable errors. The changefeed in this state keeps trying to resume until the state transfers to `Normal`. The default retry time is 30 minutes (which can be adjusted by [`changefeed-error-stuck-duration`](/ticdc/ticdc-changefeed-config.md)). If it exceeds this time, the changefeed enters a failed state. The changefeed in this state blocks GC operations. - `Finished`: The replication task is finished and has reached the preset `TargetTs`. The changefeed in this state does not block GC operations. -- `Failed`: The replication task fails. Due to some unrecoverable errors, the replication task cannot resume and cannot be recovered. To give you enough time to handle the failure, the changefeed in this state blocks GC operations. The duration of the blockage is specified by the `gc-ttl` parameter, with a default value of 24 hours. +- `Failed`: The replication task fails. The changefeed in this state does not keep trying to resume. To give you enough time to handle the failure, the changefeed in this state blocks GC operations. The duration of the blockage is specified by the `gc-ttl` parameter, with a default value of 24 hours. If the underlying issue is resolved within this duration, you can manually resume the changefeed. Otherwise, if the changefeed remains in this state beyond the `gc-ttl` duration, the replication task cannot resume and cannot be recovered. > **Note:** > -> If the changefeed encounters errors with error codes `ErrGCTTLExceeded`, `ErrSnapshotLostByGC`, or `ErrStartTsBeforeGC`, it does not block GC operations. +> - If GC is blocked by a changefeed, the changefeed will block GC advancement for up to the time specified by `gc-ttl`. After that, the changefeed will be set to the `failed` state, with an error type of `ErrGCTTLExceeded`, and will no longer block GC advancement. +> - If the changefeed encounters errors with error codes `ErrGCTTLExceeded`, `ErrSnapshotLostByGC`, or `ErrStartTsBeforeGC`, it does not block GC operations. The numbers in the preceding state transfer diagram are described as follows. @@ -35,6 +36,7 @@ The numbers in the preceding state transfer diagram are described as follows. - ⑥ The changefeed encounters an unrecoverable error and directly enters the failed state. At this time, the changefeed continues to block upstream GC for a duration specified by `gc-ttl`. - ⑦ The replication progress of the changefeed reaches the value set by `target-ts`, and the replication is completed. - ⑧ The changefeed has been suspended for a duration longer than the value specified by `gc-ttl`, thus encountering GC advancement errors, and cannot be resumed. +- ⑨ If the cause of the failure has been resolved, and the changefeed was suspended for a duration shorter than the value specified by `gc-ttl`, run the `changefeed resume` command to resume the replication task. ## Operate changefeeds @@ -42,4 +44,4 @@ You can manage a TiCDC cluster and its replication tasks using the command-line You can also use the HTTP interface (the TiCDC OpenAPI feature) to manage a TiCDC cluster and its replication tasks. For details, see [TiCDC OpenAPI](/ticdc/ticdc-open-api.md). -If your TiCDC is deployed using TiUP, you can start `cdc cli` by running the `tiup ctl:v cdc` command. Replace `v` with the TiCDC cluster version, such as `v7.4.0`. You can also run `cdc cli` directly. +If your TiCDC is deployed using TiUP, you can start `cdc cli` by running the `tiup cdc:v cli` command. Replace `v` with the TiCDC cluster version, such as `v{{{ .tidb-version }}}`. You can also run `cdc cli` directly. diff --git a/ticdc/ticdc-compatibility.md b/ticdc/ticdc-compatibility.md index f5e37e75a20f7..ff7673a002fc0 100644 --- a/ticdc/ticdc-compatibility.md +++ b/ticdc/ticdc-compatibility.md @@ -7,15 +7,26 @@ summary: Learn about compatibility issues of TiCDC and how to handle them. This section describes compatibility issues related to TiCDC and how to handle them. - +1. Create a changefeed. For more information, see [Create a replication task](/ticdc/ticdc-manage-changefeed.md#create-a-replication-task). +2. Start TiDB Lightning and import data using the logical import mode. For more information, see [Use logical import mode](/tidb-lightning/tidb-lightning-logical-import-mode-usage.md). + +In the physical import mode, TiDB Lightning imports data by inserting SST files into TiKV. TiCDC is not compatible with this mode and does not support replicating data imported through physical import mode. If you need to use both TiDB Lightning's physical import mode and TiCDC, choose one of the following solutions based on your downstream system: + +- If the downstream is a TiDB cluster, perform the following steps: + + 1. Use TiDB Lightning to import data into both the upstream and downstream TiDB clusters to ensure data consistency. + 2. Create a changefeed to replicate subsequent incremental data written through SQL. For more information, see [Create a replication task](/ticdc/ticdc-manage-changefeed.md#create-a-replication-task). + +- If the downstream is not a TiDB cluster, perform the following steps: + + 1. Use the offline import tool provided by your downstream system to import TiDB Lightning's input files. + 2. Create a changefeed to replicate subsequent incremental data written through SQL. For more information, see [Create a replication task](/ticdc/ticdc-manage-changefeed.md#create-a-replication-task). ## CLI and configuration file compatibility diff --git a/ticdc/ticdc-data-replication-capabilities.md b/ticdc/ticdc-data-replication-capabilities.md new file mode 100644 index 0000000000000..cc5bfc60414f7 --- /dev/null +++ b/ticdc/ticdc-data-replication-capabilities.md @@ -0,0 +1,64 @@ +--- +title: TiCDC Data Replication Capabilities +summary: Learn the data replication capabilities of TiCDC. +--- + +# TiCDC Data Replication Capabilities + +[TiCDC](/ticdc/ticdc-overview.md) (TiDB Change Data Capture) is a core component in the TiDB ecosystem for real-time data replication. This document provides a detailed explanation of TiCDC data replication capabilities. + +## How TiCDC works + +- TiCDC listens to TiKV change logs (Raft logs) and converts row-level data changes (`INSERT`, `UPDATE`, and `DELETE` operations) into downstream-compatible SQL statements. TiCDC does not rely on the original SQL statements executed on the upstream database. For details, see [how TiCDC processes data changes](/ticdc/ticdc-overview.md#implementation-of-processing-data-changes). + +- TiCDC generates logical operations (such as `INSERT`, `UPDATE`, and `DELETE`) equivalent to SQL semantics, rather than restoring the original SQL statements executed on the upstream database one by one. For details, see [how TiCDC processes data changes](/ticdc/ticdc-overview.md#implementation-of-processing-data-changes). + +- TiCDC ensures eventual consistency of transactions. With [redo log](/ticdc/ticdc-sink-to-mysql.md#eventually-consistent-replication-in-disaster-scenarios) enabled, TiCDC can guarantee eventual consistency in disaster recovery scenarios. With [Syncpoint](/ticdc/ticdc-upstream-downstream-check.md#enable-syncpoint) enabled, TiCDC supports consistent snapshot reads and data consistency validation. + +## Supported downstream systems + +TiCDC supports replicating data to various downstream systems, including the following: + +- [TiDB database or other MySQL-compatible databases](/ticdc/ticdc-sink-to-mysql.md) +- [Apache Kafka](/ticdc/ticdc-sink-to-kafka.md) +- Message Queue (MQ)-type sinks, such as [Pulsar](/ticdc/ticdc-sink-to-pulsar.md) +- [Storage services (Amazon S3, GCS, Azure Blob Storage, and NFS)](/ticdc/ticdc-sink-to-cloud-storage.md) +- [Snowflake, ksqlDB, SQL Server via Confluent Cloud integration](/ticdc/integrate-confluent-using-ticdc.md) +- [Apache Flink for consuming Kafka-replicated data](/replicate-data-to-kafka.md) + +## Scope of data replication + +TiCDC supports the following types of upstream data changes: + ++ **Supported:** + + - DDL and DML statements (excluding system tables). + - Index operations (`ADD INDEX`, `CREATE INDEX`): to reduce the impact on changefeed replication latency, if the downstream is TiDB, TiCDC [asynchronously executes the `ADD INDEX` and `CREATE INDEX` DDL operations](/ticdc/ticdc-ddl.md#asynchronous-execution-of-add-index-and-create-index-ddls). + - Foreign key constraint DDL statements (`ADD FOREIGN KEY`): TiCDC does **not** replicate upstream system variable settings. You need to manually configure [`foreign_key_checks`](/system-variables.md#foreign_key_checks) in the downstream to determine whether the downstream foreign key constraint check is enabled. Additionally, when writing data to the downstream, TiCDC automatically enables the session-level setting `SET SESSION foreign_key_checks = OFF;`. Therefore, even if global foreign key checks are enabled in the downstream, the data written by TiCDC will not trigger foreign key constraint validation. + ++ **Not supported**: + + - DDL and DML statements executed in upstream system tables (including `mysql.*` and `information_schema.*`). + - DDL and DML statements executed in upstream temporary tables. + - DQL (Data Query Language) and DCL (Data Control Language) statements. + +## Limitations + +- TiCDC does not support certain scenarios. For details, see [unsupported scenarios](/ticdc/ticdc-overview.md#unsupported-scenarios). +- TiCDC only verifies the integrity of upstream data changes. It does not validate whether the changes conform to upstream or downstream constraints. If the data violates downstream constraints, TiCDC will return an error when writing to the downstream. + + For example: When a changefeed is configured to filter out all DDL events, if the upstream executes a `DROP COLUMN` operation but continues to write `INSERT` statements involving that column, TiCDC will fail to replicate these DML changes to the downstream because of table schema mismatches. + +- For the TiCDC [classic architecture](https://docs.pingcap.com/tidb/stable/ticdc-classic-architecture/), when the number of tables replicated by a single TiCDC cluster exceeds the following recommended values, TiCDC might not work stably: + + | TiCDC version | Recommended number of tables to be replicated | + |---|:---:| + | v5.4.0 - v6.4.x | 2000 | + | v6.5.x - v7.4.x | 4000 | + | v7.5.x - v8.5.x | 40000 | + + > **Note:** + > + > When replicating partitioned tables, TiCDC treats each partition as a separate table. Therefore, the partition count is included when TiCDC calculates the total number of tables to be replicated. + + If the number of tables to be replicated exceeds the preceding recommended values, it is recommended to use the [TiCDC new architecture](https://docs.pingcap.com/tidb/stable/ticdc-architecture/). The new architecture supports replicating more than one million tables per changefeed, making it suitable for large-scale replication scenarios. diff --git a/ticdc/ticdc-ddl.md b/ticdc/ticdc-ddl.md index 1d05faf4d2c58..10d5ac97c77e2 100644 --- a/ticdc/ticdc-ddl.md +++ b/ticdc/ticdc-ddl.md @@ -48,7 +48,9 @@ The allow list of DDL statements supported by TiCDC is as follows: ### Asynchronous execution of `ADD INDEX` and `CREATE INDEX` DDLs -When the downstream is TiDB, TiCDC executes `ADD INDEX` and `CREATE INDEX` DDL operations asynchronously to minimize the impact on changefeed replication latency. This means that, after replicating `ADD INDEX` and `CREATE INDEX` DDLs to the downstream TiDB for execution, TiCDC returns immediately without waiting for the completion of the DDL execution. This avoids blocking subsequent DML executions. +Although TiCDC usually replicates DDL statements in order, when the downstream is TiDB, TiCDC executes DDL operations for creating and adding indexes asynchronously. This means that, after replicating `ADD INDEX` and `CREATE INDEX` DDLs to the downstream TiDB cluster for execution, TiCDC returns immediately without waiting for DDL execution to complete. This minimizes the impact on changefeed replication latency and avoids blocking subsequent DML executions. + +During the execution of the `ADD INDEX` or `CREATE INDEX` DDL operation in the downstream, when TiCDC executes the next DDL operation of the same table, this DDL operation might be blocked in the `queueing` state for a long time. This can cause TiCDC to repeatedly execute this DDL operation, and if retries take too long, it might lead to replication task failure. Starting from v7.5.4, if TiCDC has the `SUPER` permission of the downstream database, it periodically runs `ADMIN SHOW DDL JOBS` to check the status of asynchronously executed DDL tasks. TiCDC will wait for index creation to complete before proceeding with replication. Although this might increase replication latency, it avoids replication task failure. > **Note:** > @@ -77,12 +79,12 @@ TiCDC processes this type of DDL as follows: | `RENAME TABLE test.t1 TO test.t2` | Replicate | `test.t1` matches the filter rule | | `RENAME TABLE test.t1 TO ignore.t1` | Replicate | `test.t1` matches the filter rule | | `RENAME TABLE ignore.t1 TO ignore.t2` | Ignore | `ignore.t1` does not match the filter rule | -| `RENAME TABLE test.n1 TO test.t1` | Report an error and exit the replication | `test.n1` does not match the filter rule, but `test.t1` matches the filter rule. This operation is illegal. In this case, refer to the error message for handling. | +| `RENAME TABLE test.n1 TO test.t1` | Report an error and exit the replication | The old table name `test.n1` does not match the filter rule, but the new table name `test.t1` matches the filter rule. This operation is illegal. In this case, refer to the error message for handling. | | `RENAME TABLE ignore.t1 TO test.t1` | Report an error and exit the replication | Same reason as above. | #### Rename multiple tables in a DDL statement -If a DDL statement renames multiple tables, TiCDC only replicates the DDL statement when the old database name, old table names, and the new database name all match the filter rule. +If a DDL statement renames multiple tables, TiCDC replicates the DDL statement only when the **old database name**, **old table names**, and **new database name** all match the filter rule. In addition, TiCDC does not support the `RENAME TABLE` DDL that swaps the table names. The following is an example. @@ -102,6 +104,10 @@ TiCDC processes this type of DDL as follows: | `RENAME TABLE test.t1 TO ignore.t1, test.t2 TO test.t22;` | Report an error | The new database name `ignore` does not match the filter rule. | | `RENAME TABLE test.t1 TO test.t4, test.t3 TO test.t1, test.t4 TO test.t3;` | Report an error | The `RENAME TABLE` DDL swaps the names of `test.t1` and `test.t3` in one DDL statement, which TiCDC cannot handle correctly. In this case, refer to the error message for handling. | +### DDL statement considerations + +When executing cross-database DDL statements (such as `CREATE TABLE db1.t1 LIKE t2`) in the upstream, it is recommended that you explicitly specify all relevant database names in DDL statements (such as `CREATE TABLE db1.t1 LIKE db2.t2`). Otherwise, cross-database DDL statements might not be executed correctly in the downstream due to the lack of database name information. + ### SQL mode By default, TiCDC uses the default SQL mode of TiDB to parse DDL statements. If your upstream TiDB cluster uses a non-default SQL mode, you must specify the SQL mode in the TiCDC configuration file. Otherwise, TiCDC might fail to parse DDL statements correctly. For more information about TiDB SQL mode, see [SQL Mode](/sql-mode.md). @@ -124,3 +130,32 @@ CREATE TABLE "t1" ("a" int PRIMARY KEY); Because in the default SQL mode of TiDB, double quotation marks are treated as strings rather than identifiers, TiCDC fails to parse the DDL statement correctly. Therefore, when creating a replication task, it is recommended that you specify the SQL mode used by the upstream TiDB cluster in the configuration file. + +### Notes on using event filter rules to filter DDL events + +If a filtered DDL statement involves table creation or deletion, TiCDC only filters out the DDL statement without affecting the replication behavior of DML statements. The following is an example. + +Assume that the configuration file of your changefeed is as follows: + +```toml +[filter] +rules = ['test.t*'] + +matcher = ["test.t1"] # This filter rule applies only to the t1 table in the test database. +ignore-event = ["create table", "drop table", "truncate table", "rename table"] +``` + +| DDL | DDL behavior | DML behavior | Explanation | +| --- | --- | --- | --- | +| `CREATE TABLE test.t1 (id INT, name VARCHAR(50));` | Ignore | Replicate | `test.t1` matches the event filter rule, so the `CREATE TABLE` event is ignored. The replication of DML events remains unaffected. | +| `CREATE TABLE test.t2 (id INT, name VARCHAR(50));` | Replicate | Replicate | `test.t2` does not match the event filter rule. | +| `CREATE TABLE test.ignore (id INT, name VARCHAR(50));` | Ignore | Ignore | `test.ignore` matches the event filter rule, so both DDL and DML events are ignored. | +| `DROP TABLE test.t1;` | Ignore | - | `test.t1` matches the event filter rule, so the `DROP TABLE` event is ignored. Because the table is deleted, TiCDC no longer replicates DML events for `t1`. | +| `TRUNCATE TABLE test.t1;` | Ignore | Replicate | `test.t1` matches the event filter rule, so the `TRUNCATE TABLE` event is ignored. The replication of DML events remains unaffected. | +| `RENAME TABLE test.t1 TO test.t2;` | Ignore | Replicate | `test.t1` matches the event filter rule, so the `RENAME TABLE` event is ignored. The replication of DML events remains unaffected. | +| `RENAME TABLE test.t1 TO test.ignore;` | Ignore | Ignore | `test.t1` matches the event filter rule, so the `RENAME TABLE` event is ignored. `test.ignore` matches the event filter rule, so both DDL and DML events are ignored. | + +> **Note:** +> +> - When replicating data to a database, use the event filter to filter DDL events with caution. Ensure that the upstream and downstream database schemas remain consistent during replication. Otherwise, TiCDC might report errors or cause undefined replication behavior. +> - For versions earlier than v6.5.8, v7.1.4, and v7.5.1, using the event filter to filter DDL events involving table creation or deletion affects DML replication. It is not recommended to use this feature in these versions. diff --git a/ticdc/ticdc-faq.md b/ticdc/ticdc-faq.md index e787f61a5c72b..3a709fd6201c9 100644 --- a/ticdc/ticdc-faq.md +++ b/ticdc/ticdc-faq.md @@ -55,6 +55,111 @@ The expected output is as follows: > > This feature is introduced in TiCDC 4.0.3. +## How to verify if TiCDC has replicated all updates after upstream stops updating? + +After the upstream TiDB cluster stops updating, you can verify if replication is complete by comparing the latest [TSO](/glossary.md#tso) timestamp of the upstream TiDB cluster with the replication progress in TiCDC. If the TiCDC replication progress timestamp is greater than or equal to the upstream TiDB cluster's TSO, then all updates have been replicated. To verify replication completeness, perform the following steps: + +1. Get the latest TSO timestamp from the upstream TiDB cluster. + + > **Note:** + > + > Use the `TIDB_CURRENT_TSO()` function to get the current TSO, instead of using functions like `NOW()` that return the current time. + + The following example uses [`TIDB_PARSE_TSO()`](/functions-and-operators/tidb-functions.md#tidb_parse_tso) to convert the TSO to a readable time format for further comparison: + + ```sql + BEGIN; + SELECT TIDB_PARSE_TSO(TIDB_CURRENT_TSO()); + ROLLBACK; + ``` + + The output is as follows: + + ```sql + +------------------------------------+ + | TIDB_PARSE_TSO(TIDB_CURRENT_TSO()) | + +------------------------------------+ + | 2024-11-12 20:35:34.848000 | + +------------------------------------+ + ``` + +2. Get the replication progress in TiCDC. + + You can check the replication progress in TiCDC using one of the following methods: + + * **Method 1**: query the checkpoint of the changefeed (recommended). + + Use the [TiCDC command-line tool](/ticdc/ticdc-manage-changefeed.md) `cdc cli` to view the checkpoint for all replication tasks: + + ```shell + cdc cli changefeed list --server=http://127.0.0.1:8300 + ``` + + The output is as follows: + + ```json + [ + { + "id": "syncpoint", + "namespace": "default", + "summary": { + "state": "normal", + "tso": 453880043653562372, + "checkpoint": "2024-11-12 20:36:01.447", + "error": null + } + } + ] + ``` + + In the output, `"checkpoint": "2024-11-12 20:36:01.447"` indicates that TiCDC has replicated all upstream TiDB changes before this time. If this timestamp is greater than or equal to the upstream TiDB cluster's TSO obtained in step 1, then all updates have been replicated downstream. + + * **Method 2**: query Syncpoint from the downstream TiDB. + + If the downstream is a TiDB cluster and the [TiCDC Syncpoint feature](/ticdc/ticdc-upstream-downstream-check.md) is enabled, you can get the replication progress by querying the Syncpoint in the downstream TiDB. + + > **Note:** + > + > The Syncpoint update interval is controlled by the [`sync-point-interval`](/ticdc/ticdc-upstream-downstream-check.md#enable-syncpoint) configuration item. For the most up-to-date replication progress, use method 1. + + Execute the following SQL statement in the downstream TiDB to get the upstream TSO (`primary_ts`) and downstream TSO (`secondary_ts`): + + ```sql + SELECT * FROM tidb_cdc.syncpoint_v1; + ``` + + The output is as follows: + + ```sql + +------------------+------------+--------------------+--------------------+---------------------+ + | ticdc_cluster_id | changefeed | primary_ts | secondary_ts | created_at | + +------------------+------------+--------------------+--------------------+---------------------+ + | default | syncpoint | 453879870259200000 | 453879870545461257 | 2024-11-12 20:25:01 | + | default | syncpoint | 453879948902400000 | 453879949214351361 | 2024-11-12 20:30:01 | + | default | syncpoint | 453880027545600000 | 453880027751907329 | 2024-11-12 20:35:00 | + +------------------+------------+--------------------+--------------------+---------------------+ + ``` + + In the output, each row shows the upstream TiDB snapshot at `primary_ts` matches the downstream TiDB snapshot at `secondary_ts`. + + To view the replication progress, convert the latest `primary_ts` to a readable time format: + + ```sql + SELECT TIDB_PARSE_TSO(453880027545600000); + ``` + + The output is as follows: + + ```sql + +------------------------------------+ + | TIDB_PARSE_TSO(453880027545600000) | + +------------------------------------+ + | 2024-11-12 20:35:00 | + +------------------------------------+ + ``` + + If the time corresponding to the latest `primary_ts` is greater than or equal to the upstream TiDB cluster's TSO obtained in step 1, then TiCDC has replicated all updates downstream. + ## What is `gc-ttl` in TiCDC? Since v4.0.0-rc.1, PD supports external services in setting the service-level GC safepoint. Any service can register and update its GC safepoint. PD ensures that the key-value data later than this GC safepoint is not cleaned by GC. @@ -64,7 +169,7 @@ When the replication task is unavailable or interrupted, this feature ensures th When starting the TiCDC server, you can specify the Time To Live (TTL) duration of GC safepoint by configuring `gc-ttl`. You can also [use TiUP to modify](/ticdc/deploy-ticdc.md#modify-ticdc-cluster-configurations-using-tiup) `gc-ttl`. The default value is 24 hours. In TiCDC, this value means: - The maximum time the GC safepoint is retained at the PD after the TiCDC service is stopped. -- The maximum time a replication task can be suspended after the task is interrupted or manually stopped. If the time for a suspended replication task is longer than the value set by `gc-ttl`, the replication task enters the `failed` status, cannot be resumed, and cannot continue to affect the progress of the GC safepoint. +- When TiKV's GC is blocked by TiCDC's GC safepoint, `gc-ttl` indicates the maximum replication delay of a TiCDC replication task. If the delay of the replication task exceeds the value set by `gc-ttl`, the replication task enters into the `failed` state and reports the `ErrGCTTLExceeded` error. It cannot be recovered, and no longer blocks GC safepoint to advance. The second behavior above is introduced in TiCDC v4.0.13 and later versions. The purpose is to prevent a replication task in TiCDC from suspending for too long, causing the GC safepoint of the upstream TiKV cluster not to continue for a long time and retaining too many outdated data versions, thus affecting the performance of the upstream cluster. @@ -78,7 +183,13 @@ If a replication task starts after the TiCDC service starts, the TiCDC owner upd If the replication task is suspended longer than the time specified by `gc-ttl`, the replication task enters the `failed` status and cannot be resumed. The PD corresponding service GC safepoint will continue. -The Time-To-Live (TTL) that TiCDC sets for a service GC safepoint is 24 hours, which means that the GC mechanism does not delete any data if the TiCDC service can be recovered within 24 hours after it is interrupted. +The default Time-To-Live (TTL) that TiCDC sets for a service GC safepoint is 24 hours, which means that the GC mechanism does not delete the data required by TiCDC for continuing replication if the TiCDC service can be recovered within 24 hours after it is interrupted. + +## How to recover a replication task after it fails? + +1. Use `cdc cli changefeed query` to query the error information of the replication task and fix the error as soon as possible. +2. Increase the value of `gc-ttl` to allow more time to fix the error, ensuring that the replication task does not enter the `failed` status due to the replication delay exceeding `gc-ttl` after the error is fixed. +3. After evaluating the impact on the system, increase the value of [`tidb_gc_life_time`](/system-variables.md#tidb_gc_life_time-new-in-v50) in TiDB to block GC and retain data, ensuring that the replication task does not enter the `failed` status due to GC cleaning data after the error is fixed. ## How to understand the relationship between the TiCDC time zone and the time zones of the upstream/downstream databases? @@ -102,20 +213,20 @@ If you use the `cdc cli changefeed create` command without specifying the `-conf - Replicates all tables except system tables - Only replicates tables that contain [valid indexes](/ticdc/ticdc-overview.md#best-practices) -## Does TiCDC support outputting data changes in the Canal format? +## Does TiCDC support outputting data changes in the Canal protocol? -Yes. To enable Canal output, specify the protocol as `canal` in the `--sink-uri` parameter. For example: +Yes. Note that for the Canal protocol, TiCDC only supports the JSON output format, while the protobuf format is not officially supported yet. To enable Canal output, specify `protocol` as `canal-json` in the `--sink-uri` configuration. For example: {{< copyable "shell-regular" >}} ```shell -cdc cli changefeed create --server=http://127.0.0.1:8300 --sink-uri="kafka://127.0.0.1:9092/cdc-test?kafka-version=2.4.0&protocol=canal" --config changefeed.toml +cdc cli changefeed create --server=http://127.0.0.1:8300 --sink-uri="kafka://127.0.0.1:9092/cdc-test?kafka-version=2.4.0&protocol=canal-json" --config changefeed.toml ``` > **Note:** > > * This feature is introduced in TiCDC 4.0.2. -> * TiCDC currently supports outputting data changes in the Canal format only to MQ sinks such as Kafka. +> * TiCDC currently supports outputting data changes in the Canal-JSON format only to MQ sinks such as Kafka. For more information, refer to [TiCDC changefeed configurations](/ticdc/ticdc-changefeed-config.md). @@ -130,7 +241,7 @@ For more information, refer to [TiCDC changefeed configurations](/ticdc/ticdc-ch ## When TiCDC replicates data to Kafka, can I control the maximum size of a single message in TiDB? -When `protocol` is set to `avro` or `canal-json`, messages are sent per row change. A single Kafka message contains only one row change and is generally no larger than Kafka's limit. Therefore, there is no need to limit the size of a single message. If the size of a single Kafka message does exceed Kakfa's limit, refer to [Why does the latency from TiCDC to Kafka become higher and higher?](/ticdc/ticdc-faq.md#why-does-the-latency-from-ticdc-to-kafka-become-higher-and-higher). +When `protocol` is set to `avro` or `canal-json`, messages are sent per row change. A single Kafka message contains only one row change and is generally no larger than Kafka's limit. Therefore, there is no need to limit the size of a single message. If the size of a single Kafka message does exceed Kafka's limit, refer to [Why does the latency from TiCDC to Kafka become higher and higher?](/ticdc/ticdc-faq.md#why-does-the-latency-from-ticdc-to-kafka-become-higher-and-higher). When `protocol` is set to `open-protocol`, messages are sent in batches. Therefore, one Kafka message might be excessively large. To avoid this situation, you can configure the `max-message-bytes` parameter to control the maximum size of data sent to the Kafka broker each time (optional, `10MB` by default). You can also configure the `max-batch-size` parameter (optional, `16` by default) to specify the maximum number of change records in each Kafka message. @@ -181,7 +292,15 @@ For more information, refer to [Open protocol Row Changed Event format](/ticdc/t ## How much PD storage does TiCDC use? -TiCDC uses etcd in PD to store and regularly update the metadata. Because the time interval between the MVCC of etcd and PD's default compaction is one hour, the amount of PD storage that TiCDC uses is proportional to the amount of metadata versions generated within this hour. However, in v4.0.5, v4.0.6, and v4.0.7, TiCDC has a problem of frequent writing, so if there are 1000 tables created or scheduled in an hour, it then takes up all the etcd storage and returns the `etcdserver: mvcc: database space exceeded` error. You need to clean up the etcd storage after getting this error. See [etcd maintenance space-quota](https://etcd.io/docs/v3.4.0/op-guide/maintenance/#space-quota) for details. It is recommended to upgrade your cluster to v4.0.9 or later versions. +When using TiCDC, you might encounter the `etcdserver: mvcc: database space exceeded` error, which is primarily related to the mechanism that TiCDC uses etcd in PD to store metadata. + +etcd uses Multi-Version Concurrency Control (MVCC) to store data, and the default compaction interval in PD is 1 hour. This means that etcd retains multiple versions of all data for 1 hour before compaction. + +Before v6.0.0, TiCDC uses etcd in PD to store and update metadata for all tables in a changefeed. Therefore, the PD storage space used by TiCDC is proportional to the number of tables being replicated by the changefeed. When TiCDC is replicating a large number of tables, the etcd storage space could fill up quickly, increasing the probability of the `etcdserver: mvcc: database space exceeded` error. + +If you encounter this error, refer to [etcd maintenance space-quota](https://etcd.io/docs/v3.4.0/op-guide/maintenance/#space-quota) to clean up the etcd storage space. + +Starting from v6.0.0, TiCDC optimizes its metadata storage mechanism, effectively avoiding the etcd storage space issues caused by the preceding reasons. If your TiCDC version is earlier than v6.0.0, it is recommended to upgrade to v6.0.0 or later versions. ## Does TiCDC support replicating large transactions? Is there any risk? @@ -242,7 +361,9 @@ Since v5.0.1 or v4.0.13, for each replication to MySQL, TiCDC automatically sets TiCDC guarantees that all data is replicated at least once. When there is duplicate data in the downstream, write conflicts occur. To avoid this problem, TiCDC converts `INSERT` and `UPDATE` statements into `REPLACE INTO` statements. This behavior is controlled by the `safe-mode` parameter. -In versions earlier than v6.1.3, `safe-mode` defaults to `true`, which means all `INSERT` and `UPDATE` statements are converted into `REPLACE INTO` statements. In v6.1.3 and later versions, TiCDC can automatically determine whether the downstream has duplicate data, and the default value of `safe-mode` changes to `false`. If no duplicate data is detected, TiCDC replicates `INSERT` and `UPDATE` statements without conversion. +In versions earlier than v6.1.3, the default value of `safe-mode` is `true`, which means all `INSERT` and `UPDATE` statements are converted into `REPLACE INTO` statements. + +In v6.1.3 and later versions, the default value of `safe-mode` changes to `false`, and TiCDC can automatically determine whether the downstream has duplicate data. If no duplicate data is detected, TiCDC directly replicates `INSERT` and `UPDATE` statements without conversion; otherwise, TiCDC converts `INSERT` and `UPDATE` statements into `REPLACE INTO` statements and then replicates them. ## When the sink of the replication downstream is TiDB or MySQL, what permissions do users of the downstream database need? @@ -264,9 +385,21 @@ If you need to replicate `recover table` to the downstream TiDB, you should have When upstream write traffic is at peak hours, the downstream may fail to consume all data in a timely manner, resulting in data pile-up. TiCDC uses disks to process the data that is piled up. TiCDC needs to write data to disks during normal operation. However, this is not usually the bottleneck for replication throughput and replication latency, given that writing to disks only results in latency within a hundred milliseconds. TiCDC also uses memory to accelerate reading data from disks to improve replication performance. -## Why does replication using TiCDC stall or even stop after data restore using TiDB Lightning and BR from upstream? +## Why does replication using TiCDC stall or even stop after data restore using TiDB Lightning physical import mode and BR from upstream? + +Currently, TiCDC is not yet fully compatible with [TiDB Lightning physical import mode](/tidb-lightning/tidb-lightning-physical-import-mode.md) and BR. Therefore, avoid using TiDB Lightning physical import mode and BR on tables that are replicated by TiCDC. Otherwise, unknown errors might occur, such as TiCDC replication getting stuck, a significant spike in replication latency, or data loss. + +If you need to use TiDB Lightning physical import mode or BR to restore data for some tables replicated by TiCDC, take these steps: + +1. Remove the TiCDC replication task related to these tables. -Currently, TiCDC is not yet fully compatible with TiDB Lightning and BR. Therefore, please avoid using TiDB Lightning and BR on tables that are replicated by TiCDC. +2. Use TiDB Lightning physical import mode or BR to restore data separately in the upstream and downstream clusters of TiCDC. + +3. After the restoration is complete and data consistency between the upstream and downstream clusters is verified, create a new TiCDC replication task for incremental replication, with the timestamp (TSO) from the upstream backup as the `start-ts` for the task. For example, assuming the snapshot timestamp of the BR backup in the upstream cluster is `431434047157698561`, you can create a new TiCDC replication task using the following command: + + ```shell + cdc cli changefeed create -c "upstream-to-downstream-some-tables" --start-ts=431434047157698561 --sink-uri="mysql://root@127.0.0.1:4000? time-zone=" + ``` ## After a changefeed resumes from pause, its replication latency gets higher and higher and returns to normal only after a few minutes. Why? @@ -278,11 +411,13 @@ For TiCDC versions earlier than v6.5.2, it is recommended that you deploy TiCDC ## What is the order of executing DML and DDL statements? -Currently, TiCDC adopts the following order: +For most DDL statements, TiCDC adopts the following order: + +1. TiCDC blocks the replication progress of the tables affected by DDL statements until the DDL `commitTS`. This ensures that DML statements executed before DDL `commitTS` can be successfully replicated to the downstream first. +2. TiCDC continues with the replication of DDL statements. If there are multiple DDL statements, TiCDC usually replicates them in a serial manner. +3. After the DDL statements are executed in the downstream, TiCDC continues with the replication of DML statements executed after DDL `commitTS`. -1. TiCDC blocks the replication progress of the tables affected by DDL statements until the DDL `CommiTs`. This ensures that DML statements executed before DDL `CommiTs` can be successfully replicated to the downstream first. -2. TiCDC continues with the replication of DDL statements. If there are multiple DDL statements, TiCDC replicates them in a serial manner. -3. After the DDL statements are executed in the downstream, TiCDC will continue with the replication of DML statements executed after DDL `CommiTs`. +For `ADD INDEX` and `CREATE INDEX`, when the downstream is TiDB, TiCDC executes these DDLs asynchronously and does not wait for them to finish executing in the downstream before returning, to minimize the impact on changefeed replication latency. For more information, see [Asynchronous execution of `ADD INDEX` and `CREATE INDEX` DDLs](/ticdc/ticdc-ddl.md#asynchronous-execution-of-add-index-and-create-index-ddls). ## How should I check whether the upstream and downstream data is consistent? @@ -330,3 +465,73 @@ This is because the default port number of the TiCDC cluster deployed by TiDB Op } ] ``` + +## Does TiCDC replicate generated columns of DML operations? + +Generated columns include virtual generated columns and stored generated columns. TiCDC ignores virtual generated columns and only replicates stored generated columns to the downstream. However, stored generated columns are also ignored when the downstream is MySQL or another MySQL-compatible database (rather than Kafka or other storage services). + +> **Note:** +> +> When replicating stored generated columns to Kafka or a storage service and then writing them back to MySQL, `Error 3105 (HY000): The value specified for generated column 'xx' in table 'xxx' is not allowed` might occur. To avoid this error, you can use [Open Protocol](/ticdc/ticdc-open-protocol.md#ticdc-open-protocol) for replication. The output of this protocol includes [bit flags of columns](/ticdc/ticdc-open-protocol.md#bit-flags-of-columns), which can distinguish whether a column is a generated column. + +## How do I resolve frequent `CDC:ErrMySQLDuplicateEntryCDC` errors? + +When using TiCDC to replicate data to TiDB or MySQL, you might encounter the following error if SQL statements in the upstream are executed in a specific pattern: + +`CDC:ErrMySQLDuplicateEntryCDC` + +The cause of the error: TiDB combines `DELETE + INSERT` operations on the same row within the same transaction into a single `UPDATE` row change. When TiCDC replicates these changes as updates to the downstream, the `UPDATE` operations attempting to swap unique key values might result in conflicts. + +Taking the following table as an example: + +```sql +CREATE TABLE data_table ( + id BIGINT(20) NOT NULL PRIMARY KEY, + value BINARY(16) NOT NULL, + UNIQUE KEY value_index (value) +) CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +``` + +If the upstream attempts to swap the `value` field of the two rows in the table: + +```sql +DELETE FROM data_table WHERE id = 1; +DELETE FROM data_table WHERE id = 2; +INSERT INTO data_table (id, value) VALUES (1, 'v3'); +INSERT INTO data_table (id, value) VALUES (2, 'v1'); +``` + +TiDB generates two `UPDATE` row changes, so TiCDC converts them into two `UPDATE` statements for replication to the downstream: + +```sql +UPDATE data_table SET value = 'v3' WHERE id = 1; +UPDATE data_table SET value = 'v1' WHERE id = 2; +``` + +If the downstream table still contains `v1` when executing the second `UPDATE` statement, it violates the unique key constraint on the `value` column, resulting in the `CDC:ErrMySQLDuplicateEntryCDC` error. + +If the `CDC:ErrMySQLDuplicateEntryCDC` error occurs frequently, you can enable TiCDC safe mode by setting the `safe-mode=true` parameter in the [`sink-uri`](/ticdc/ticdc-sink-to-mysql.md#configure-sink-uri-for-mysql-or-tidb) configuration: + +``` +mysql://user:password@host:port/?safe-mode=true +``` + +In safe mode, TiCDC splits the `UPDATE` operation into `DELETE + REPLACE INTO` for execution, thus avoiding the unique key conflict error. + +## Why do TiCDC replication tasks to Kafka often fail with `broken pipe` errors? + +TiCDC uses the Sarama client to replicate data to Kafka. To avoid out-of-order data, TiCDC disables the automatic retry mechanism of Sarama (by setting the retry count to 0). As a result, if the connection between TiCDC and Kafka is closed by Kafka after being idle for some time, subsequent writes from TiCDC will trigger a `write: broken pipe` error, causing the replication task to fail. + +Although the changefeed might fail due to this error, TiCDC automatically restarts the affected changefeed, so the replication task can continue running normally. Note that during the restart process, the replication latency (lag) of the changefeed might temporarily increase by a small amount, typically within 30 seconds, before automatically returning to normal. + +If your application is highly sensitive to changefeed latency, it is recommended to do the following: + +1. Increase the Kafka connection idle timeout in the Kafka broker configuration file. For example: + + ```properties + connections.max.idle.ms=86400000 # Set to 1 day + ``` + + It is recommended to adjust the value of `connections.max.idle.ms` according to your actual replication workload. For example, if a TiCDC changefeed always replicates data within several minutes, you can set `connections.max.idle.ms` to several minutes instead of a very large value. + +2. Restart Kafka to apply the configuration change. This prevents connections from being closed prematurely and helps reduce `broken pipe` errors. \ No newline at end of file diff --git a/ticdc/ticdc-filter.md b/ticdc/ticdc-filter.md index 6ad1c17ec9250..449152c040613 100644 --- a/ticdc/ticdc-filter.md +++ b/ticdc/ticdc-filter.md @@ -55,6 +55,22 @@ ignore-update-new-value-expr = "gender = 'male' and age > 18" # Ignore update DM Description of configuration parameters: - `matcher`: the database and table that this event filter rule applies to. The syntax is the same as [table filter](/table-filter.md). + + > **Note:** + > + > `matcher` matches the database name, so you need to pay extra attention when configuring it. For example, when the `event-filters` configuration is as follows: + > + > ```toml + > [filter] + > [[filter.event-filters]] + > matcher = ["test.t1"] + > ignore-sql = ["^drop"] + > ``` + > + > `ignore-sql = ["^drop"]` not only filters out `DROP TABLE test.t1` but also filters out `DROP DATABASE test`, because `matcher` contains the database name `test`. + > + > If you only want to filter out the specified table instead of the entire database, modify the `ignore-sql` value to `["drop table"]`. + - `ignore-event`: the event type to be ignored. This parameter accepts an array of strings. You can configure multiple event types. Currently, the following event types are supported: | Event | Type | Alias | Description | diff --git a/ticdc/ticdc-manage-changefeed.md b/ticdc/ticdc-manage-changefeed.md index 3f548732016cb..a2b201a6f694f 100644 --- a/ticdc/ticdc-manage-changefeed.md +++ b/ticdc/ticdc-manage-changefeed.md @@ -1,7 +1,6 @@ --- title: Manage Changefeeds summary: Learn how to manage TiCDC changefeeds. -aliases: ['/tidb/dev/manage-ticdc'] --- # Manage Changefeeds @@ -19,7 +18,7 @@ cdc cli changefeed create --server=http://10.0.10.25:8300 --sink-uri="mysql://ro ```shell Create changefeed successfully! ID: simple-replication-task -Info: {"upstream_id":7178706266519722477,"namespace":"default","id":"simple-replication-task","sink_uri":"mysql://root:xxxxx@127.0.0.1:4000/?time-zone=","create_time":"2022-12-19T15:05:46.679218+08:00","start_ts":438156275634929669,"engine":"unified","config":{"case_sensitive":true,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":3600000000000,"filter":{"rules":["test.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"none","enable_partition_separator":false},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"storage":""}},"state":"normal","creator_version":"v6.5.0"} +Info: {"upstream_id":7178706266519722477,"namespace":"default","id":"simple-replication-task","sink_uri":"mysql://root:xxxxx@127.0.0.1:4000/?time-zone=","create_time":"{{{ .tidb-release-date }}}T15:05:46.679218+08:00","start_ts":438156275634929669,"engine":"unified","config":{"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":3600000000000,"filter":{"rules":["test.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"none","enable_partition_separator":false},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"storage":""}},"state":"normal","creator_version":"v{{{ .tidb-version }}}"} ``` ## Query the replication task list @@ -81,7 +80,7 @@ cdc cli changefeed query --server=http://10.0.10.25:8300 --changefeed-id=simple- ```shell { "info": { - "sink-uri": "mysql://127.0.0.1:3306/?max-txn-row=20\u0026worker-number=4", + "sink-uri": "mysql://127.0.0.1:3306/?max-txn-row=20\u0026worker-count=4", "opts": {}, "create-time": "2020-08-27T10:33:41.687983832+08:00", "start-ts": 419036036249681921, @@ -90,7 +89,7 @@ cdc cli changefeed query --server=http://10.0.10.25:8300 --changefeed-id=simple- "sort-engine": "unified", "sort-dir": ".", "config": { - "case-sensitive": true, + "case-sensitive": false, "filter": { "rules": [ "*.*" @@ -136,7 +135,7 @@ cdc cli changefeed query --server=http://10.0.10.25:8300 --changefeed-id=simple- } ``` -In the preceding command and result: +In the preceding command and result: - `info` is the replication configuration of the queried changefeed. - `status` is the replication state of the queried changefeed. @@ -144,7 +143,7 @@ In the preceding command and result: - `checkpoint-ts`: The largest transaction `TS` in the current `changefeed`. Note that this `TS` has been successfully written to the downstream. - `admin-job-type`: The status of a changefeed: - `0`: The state is normal. - - `1`: The task is paused. When the task is paused, all replicated `processor`s exit. The configuration and the replication status of the task are retained, so you can resume the task from `checkpiont-ts`. + - `1`: The task is paused. When the task is paused, all replicated `processor`s exit. The configuration and the replication status of the task are retained, so you can resume the task from `checkpoint-ts`. - `2`: The task is resumed. The replication task resumes from `checkpoint-ts`. - `3`: The task is removed. When the task is removed, all replicated `processor`s are ended, and the configuration information of the replication task is cleared up. Only the replication status is retained for later queries. - `task-status` indicates the state of each replication sub-task in the queried changefeed. @@ -170,7 +169,7 @@ cdc cli changefeed resume --server=http://10.0.10.25:8300 --changefeed-id simple ``` - `--changefeed-id=uuid` represents the ID of the changefeed that corresponds to the replication task you want to resume. -- `--overwrite-checkpoint-ts`: starting from v6.2.0, you can specify the starting TSO of resuming the replication task. TiCDC starts pulling data from the specified TSO. The argument accepts `now` or a specific TSO (such as 434873584621453313). The specified TSO must be in the range of (GC safe point, CurrentTSO]. If this argument is not specified, TiCDC replicates data from the current `checkpoint-ts` by default. +- `--overwrite-checkpoint-ts`: starting from v6.2.0, you can specify the starting TSO of resuming the replication task. TiCDC starts pulling data from the specified TSO. The argument accepts `now` or a specific TSO (such as 434873584621453313). The specified TSO must be in the range of (GC safe point, CurrentTSO]. If this argument is not specified, TiCDC replicates data from the current `checkpoint-ts` by default. You can use the `cdc cli changefeed list` command to check the current value of `checkpoint-ts`. - `--no-confirm`: when the replication is resumed, you do not need to confirm the related information. Defaults to `false`. > **Note:** @@ -196,7 +195,7 @@ TiCDC supports modifying the configuration of the replication task (not dynamica ```shell cdc cli changefeed pause -c test-cf --server=http://10.0.10.25:8300 -cdc cli changefeed update -c test-cf --server=http://10.0.10.25:8300 --sink-uri="mysql://127.0.0.1:3306/?max-txn-row=20&worker-number=8" --config=changefeed.toml +cdc cli changefeed update -c test-cf --server=http://10.0.10.25:8300 --sink-uri="mysql://127.0.0.1:3306/?max-txn-row=20&worker-count=8" --config=changefeed.toml cdc cli changefeed resume -c test-cf --server=http://10.0.10.25:8300 ``` @@ -287,10 +286,10 @@ For the changefeeds created using `cdc cli` after v4.0.13, Unified Sorter is ena To check whether or not the Unified Sorter feature is enabled on a changefeed, you can run the following example command (assuming the IP address of the PD instance is `http://10.0.10.25:2379`): ```shell -cdc cli --server="http://10.0.10.25:8300" changefeed query --changefeed-id=simple-replication-task | grep 'sort-engine' +cdc cli --server="http://10.0.10.25:8300" changefeed query --changefeed-id=simple-replication-task | grep 'sort_engine' ``` -In the output of the above command, if the value of `sort-engine` is "unified", it means that Unified Sorter is enabled on the changefeed. +In the output of the above command, if the value of `sort_engine` is "unified", it means that Unified Sorter is enabled on the changefeed. > **Note:** > diff --git a/ticdc/ticdc-open-api-v2.md b/ticdc/ticdc-open-api-v2.md index 4596163ed650b..4e5a12029016d 100644 --- a/ticdc/ticdc-open-api-v2.md +++ b/ticdc/ticdc-open-api-v2.md @@ -92,7 +92,7 @@ curl -X GET http://127.0.0.1:8300/api/v2/status ```json { - "version": "v7.4.0", + "version": "v{{{ .tidb-version }}}", "git_hash": "10413bded1bdb2850aa6d7b94eb375102e9c44dc", "id": "d2912e63-3349-447c-90ba-72a4e04b5e9e", "pid": 1447, @@ -147,7 +147,7 @@ This interface is used to submit a replication task to TiCDC. If the request is "changefeed_id": "string", "replica_config": { "bdr_mode": true, - "case_sensitive": true, + "case_sensitive": false, "check_gc_safe_point": true, "consistent": { "flush_interval": 0, @@ -155,7 +155,6 @@ This interface is used to submit a replication task to TiCDC. If the request is "max_log_size": 0, "storage": "string" }, - "enable_old_value": true, "enable_sync_point": true, "filter": { "do_dbs": [ @@ -266,7 +265,7 @@ The descriptions of the `replica_config` parameters are as follows. | Parameter name | Description | | :------------------------ | :----------------------------------------------------- | | `bdr_mode` | `BOOLEAN` type. Determines whether to enable [bidirectional replication](/ticdc/ticdc-bidirectional-replication.md). The default value is `false`. (Optional) | -| `case_sensitive` | `BOOLEAN` type. Determines whether to be case-sensitive when filtering table names. The default value is `true`. (Optional) | +| `case_sensitive` | `BOOLEAN` type. Determines whether to be case-sensitive when filtering table names. Starting from v7.5.0, the default value changes from `true` to `false`. (Optional) | | `check_gc_safe_point` | `BOOLEAN` type. Determines whether to check that the start time of the replication task is earlier than the GC time. The default value is `true`. (Optional) | | `consistent` | The configuration parameters of redo log. (Optional) | | `enable_sync_point` | `BOOLEAN` type. Determines whether to enable `sync point`. (Optional) | @@ -287,6 +286,11 @@ The `consistent` parameters are described as follows: | `level` | `STRING` type. The consistency level of the replicated data. (Optional) | | `max_log_size` | `UINT64` type. The maximum value of redo log. (Optional) | | `storage` | `STRING` type. The destination address of the storage. (Optional) | +| `use_file_backend` | `BOOL` type. Specifies whether to store the redo log in a local file. (Optional) | +| `encoding_worker_num` | `INT` type. The number of encoding and decoding workers in the redo module. (Optional) | +| `flush_worker_num` | `INT` type. The number of flushing workers in the redo module. (Optional) | +| `compression` | `STRING` type. The behavior to compress redo log files. Available options are `""` and `"lz4"`. The default value is `""`, which means no compression. (Optional) | +| `flush_concurrency` | `INT` type. The concurrency for uploading a single file. The default value is `1`, which means concurrency is disabled. (Optional) The `filter` parameters are described as follows: @@ -327,11 +331,13 @@ The `sink` parameters are described as follows: | `date_separator` | `STRING` type. Indicates the date separator type of the file directory. Value options are `none`, `year`, `month`, and `day`. `none` is the default value and means that the date is not separated. (Optional) | | `dispatchers` | An configuration array for event dispatching. (Optional) | | `encoder_concurrency` | `INT` type. The number of encoder threads in the MQ sink. The default value is `16`. (Optional) | -| `protocol` | `STRING` type. For MQ sinks, you can specify the protocol format of the message. The following protocols are currently supported: `canal-json`, `open-protocol`, `canal`, `avro`, and `maxwell`. | +| `protocol` | `STRING` type. For MQ sinks, you can specify the protocol format of the message. The following protocols are currently supported: `canal-json`, `open-protocol`, and `avro`. | | `schema_registry` | `STRING` type. The schema registry address. (Optional) | | `terminator` | `STRING` type. The terminator is used to separate two data change events. The default value is null, which means `"\r\n"` is used as the terminator. (Optional) | | `transaction_atomicity` | `STRING` type. The atomicity level of the transaction. (Optional) | | `only_output_updated_columns` | `BOOLEAN` type. For MQ sinks using the `canal-json` or `open-protocol` protocol, you can specify whether only output the modified columns. The default value is `false`. (Optional) | +| `cloud_storage_config` | The storage sink configuration. (Optional) | +| `open` | The Open Protocol configuration. (Optional, introduced in v7.5.2) | `sink.column_selectors` is an array. The parameters are described as follows: @@ -344,16 +350,17 @@ The `sink.csv` parameters are described as follows: | Parameter name | Description | |:-----------------|:---------------------------------------| -| `delimiter` | `STRING` type. The character used to separate fields in the CSV file. The value must be an ASCII character and defaults to `,`. | -| `include_commit_ts` | `BOOLEAN` type. Whether to include commit-ts in CSV rows. The default value is `false`. | -| `null` | `STRING` type. The character that is displayed when a CSV column is null. The default value is `\N`. | -| `quote` | `STRING` type. The quotation character used to surround fields in the CSV file. If the value is empty, no quotation is used. The default value is `"`. | +| `delimiter` | `STRING` type. The character used to separate fields in the CSV file. The value must be an ASCII character and defaults to `,`. | +| `include_commit_ts` | `BOOLEAN` type. Whether to include commit-ts in CSV rows. The default value is `false`. | +| `null` | `STRING` type. The character that is displayed when a CSV column is null. The default value is `\N`. | +| `quote` | `STRING` type. The quotation character used to surround fields in the CSV file. If the value is empty, no quotation is used. The default value is `"`. | +| `binary_encoding_method` | `STRING` type. The encoding method of binary data, which can be `"base64"` or `"hex"`. The default value is `"base64"`. | -`sink.dispatchers`: for the sink of MQ type, you can use this parameter to configure the event dispatcher. The following dispatchers are supported: `default`, `ts`, `rowid`, and `table`. The dispatcher rules are as follows: +`sink.dispatchers`: for the sink of MQ type, you can use this parameter to configure the event dispatcher. The following dispatchers are supported: `default`, `ts`, `index-value`, and `table`. The dispatcher rules are as follows: - `default`: dispatches events in the `table` mode. - `ts`: uses the commitTs of the row change to create the hash value and dispatch events. -- `rowid`: uses the name and value of the selected HandleKey column to create the hash value and dispatch events. +- `index-value`: uses the name and value of the selected HandleKey column to create the hash value and dispatch events. - `table`: uses the schema name of the table and the table name to create the hash value and dispatch events. `sink.dispatchers` is an array. The parameters are described as follows: @@ -364,12 +371,30 @@ The `sink.csv` parameters are described as follows: | `partition` | `STRING` type. The target partition for dispatching events. | | `topic` | `STRING` type. The target topic for dispatching events. | +`sink.cloud_storage_config` parameters are described as follows: + +| Parameter name | Description | +|:-----------------|:---------------------------------------| +| `worker_count` | `INT` type. The concurrency for saving data changes to the downstream cloud storage. | +| `flush_interval` | `STRING` type. The interval for saving data changes to the downstream cloud storage. | +| `file_size` | `INT` type. A data change file is saved to the cloud storage when the number of bytes in this file exceeds the value of this parameter. | +| `file_expiration_days` | `INT` type. The duration to retain files, which takes effect only when `date-separator` is configured as `day`. | +| `file_cleanup_cron_spec` | `STRING` type. The running cycle of the scheduled cleanup task, compatible with the crontab configuration, with a format of ` `. | +| `flush_concurrency` | `INT` type. The concurrency for uploading a single file. | +| `output_raw_change_event` | `BOOLEAN` type. Controls whether to output the original data change event for a non-MySQL sink. | + +`sink.open` parameters are described as follows: + +| Parameter name | Description | +|:-------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `output_old_value` | `BOOLEAN` type. It controls whether to output the value before the row data changes. The default value is `true`. When it is disabled, the UPDATE event does not output the "p" field. | + ### Example The following request creates a replication task with an ID of `test5` and `sink_uri` of `blackhome://`. ```shell -curl -X POST -H "'Content-type':'application/json'" http://127.0.0.1:8300/api/v2/changefeeds -d '{"changefeed_id":"test5","sink_uri":"blackhole://"}' +curl -X POST -H "Content-type: application/json" http://127.0.0.1:8300/api/v2/changefeeds -d '{"changefeed_id":"test5","sink_uri":"blackhole://"}' ``` If the request is successful, `200 OK` is returned. If the request fails, an error message and error code are returned. @@ -383,7 +408,7 @@ If the request is successful, `200 OK` is returned. If the request fails, an err "checkpoint_ts": 0, "config": { "bdr_mode": true, - "case_sensitive": true, + "case_sensitive": false, "check_gc_safe_point": true, "consistent": { "flush_interval": 0, @@ -391,7 +416,6 @@ If the request is successful, `200 OK` is returned. If the request fails, an err "max_log_size": 0, "storage": "string" }, - "enable_old_value": true, "enable_sync_point": true, "filter": { "do_dbs": [ @@ -587,7 +611,7 @@ To modify the changefeed configuration, follow the steps of `pause the replicati { "replica_config": { "bdr_mode": true, - "case_sensitive": true, + "case_sensitive": false, "check_gc_safe_point": true, "consistent": { "flush_interval": 0, @@ -595,7 +619,6 @@ To modify the changefeed configuration, follow the steps of `pause the replicati "max_log_size": 0, "storage": "string" }, - "enable_old_value": true, "enable_sync_point": true, "filter": { "do_dbs": [ @@ -703,7 +726,7 @@ The meanings of the above parameters are the same as those in the [Create a repl The following request updates the `target_ts` of the replication task with the ID `test1` to `32`. ```shell - curl -X PUT -H "'Content-type':'application/json'" http://127.0.0.1:8300/api/v2/changefeeds/test1 -d '{"target_ts":32}' +curl -X PUT -H "Content-type: application/json" http://127.0.0.1:8300/api/v2/changefeeds/test1 -d '{"target_ts":32}' ``` If the request is successful, `200 OK` is returned. If the request fails, an error message and error code are returned. The meanings of the JSON response body are the same as those in the [Create a replication task](#create-a-replication-task) section. See that section for details. @@ -792,6 +815,95 @@ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test1 The meanings of the JSON response body are the same as those in the [Create a replication task](#create-a-replication-task) section. See that section for details. +## Query whether a specific replication task is completed + +This API is a synchronous interface. If the request is successful, it returns the synchronization status of the specified replication task (changefeed), including whether the task is completed and additional details. + +### Request URI + +`GET /api/v2/changefeed/{changefeed_id}/synced` + +### Parameter description + +#### Path parameter + +| Parameter name | Description | +|:----------------|:----------------------------| +| `changefeed_id` | The ID of the replication task (changefeed) to be queried. | + +### Examples + +The following request queries the synchronization status of the replication task with the ID `test1`. + +```shell +curl -X GET http://127.0.0.1:8300/api/v2/changefeed/test1/synced +``` + +**Example 1: The synchronization is completed** + +```json +{ + "synced": true, + "sink_checkpoint_ts": "2023-11-30 15:14:11.015", + "puller_resolved_ts": "2023-11-30 15:14:12.215", + "last_synced_ts": "2023-11-30 15:08:35.510", + "now_ts": "2023-11-30 15:14:11.511", + "info": "Data syncing is finished" +} +``` + +The response includes the following fields: + +- `synced`: whether this replication task is completed. `true` means the task is completed, and `false` means potential incompleteness. If it is `false`, you need to check both the `info` field and other fields for the specific status. +- `sink_checkpoint_ts`: the checkpoint-ts value of the sink module, in PD time. +- `puller_resolved_ts`: the resolved-ts value of the puller module, in PD time. +- `last_synced_ts`: the commit-ts value of the latest data processed by TiCDC, in PD time. +- `now_ts`: current PD time. +- `info`: supplementary information to assist in determining the synchronization status, especially when `synced` is `false`. + +**Example 2: The synchronization is not completed** + +```json +{ + "synced": false, + "sink_checkpoint_ts": "2023-11-30 15:26:31.519", + "puller_resolved_ts": "2023-11-30 15:26:23.525", + "last_synced_ts": "2023-11-30 15:24:30.115", + "now_ts": "2023-11-30 15:26:31.511", + "info": "The data syncing is not finished, please wait" +} +``` + +This example shows the response of an ongoing replication task. By checking both `synced` and `info` fields, you can learn that the replication task is not completed yet and further waiting is expected. + +**Example 3: The synchronization status needs further check** + +```json +{ + "synced":false, + "sink_checkpoint_ts":"2023-12-13 11:45:13.515", + "puller_resolved_ts":"2023-12-13 11:45:13.525", + "last_synced_ts":"2023-12-13 11:45:07.575", + "now_ts":"2023-12-13 11:50:24.875", + "info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view 'TiKV-Details' > 'Resolved-Ts' > 'Max Leader Resolved TS gap' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait" +} +``` + +This API enables you to query the synchronization status even when the upstream cluster encounters disasters. In certain situations, you might not be able to directly determine whether the current data replication task of TiCDC is completed or not. In such cases, you can request this API, and then check both the `info` field in the response and the current status of the upstream cluster to determine the specific status. + +In this example, `sink_checkpoint_ts` is behind `now_ts` in time, either because TiCDC is still catching up with data replication, or because the PD or TiKV has failed. If this is due to TiCDC still catching up with data replication, it means that the replication task is not completed yet. If this is due to a PD or TiKV failure, it means that the replication task is completed. Therefore, you need to check the `info` field to assist in determining the cluster status. + +**Example 4: Query error** + +```json +{ + "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", + "error_code": "CDC:ErrPDEtcdAPIError" +} +``` + +In cases where PD in the upstream cluster fails for a long period of time, querying this API might return an error similar to the preceding one, which provides no information for further check. Because PD failures directly affect TiCDC data replication, when getting such errors, you can assume that TiCDC has completed the data replication as much as possible, but data loss might still occur in the downstream cluster due to PD failures. + ## Pause a replication task This API pauses a replication task. If the request is successful, `200 OK` is returned. The returned result only means that the server agrees to run the command but does not guarantee that the command will be run successfully. @@ -1004,7 +1116,7 @@ This API is a synchronous interface. If the request is successful, `200 OK` is r ### Example ```shell -curl -X POST -H "'Content-type':'application/json'" http://127.0.0.1:8300/api/v2/log -d '{"log_level":"debug"}' +curl -X POST -H "Content-type: application/json" http://127.0.0.1:8300/api/v2/log -d '{"log_level":"debug"}' ``` If the request is successful, `200 OK` is returned. If the request fails, an error message and error code are returned. diff --git a/ticdc/ticdc-open-api.md b/ticdc/ticdc-open-api.md index 2194e7752ebe9..e7d141f863344 100644 --- a/ticdc/ticdc-open-api.md +++ b/ticdc/ticdc-open-api.md @@ -148,22 +148,22 @@ The configuration parameters of sink are as follows: { "dispatchers":[ {"matcher":["test1.*", "test2.*"], "dispatcher":"ts"}, - {"matcher":["test3.*", "test4.*"], "dispatcher":"rowid"} + {"matcher":["test3.*", "test4.*"], "dispatcher":"index-value"} ], - "protocal":"canal-json" + "protocol":"canal-json" } ``` -`dispatchers`: For the sink of MQ type, you can use dispatchers to configure the event dispatcher. Four dispatchers are supported: `default`, `ts`, `rowid`, and `table`. The dispatcher rules are as follows: +`dispatchers`: For the sink of MQ type, you can use dispatchers to configure the event dispatcher. Four dispatchers are supported: `default`, `ts`, `index-value`, and `table`. The dispatcher rules are as follows: - `default`: dispatches events in the `table` mode. - `ts`: uses the commitTs of the row change to create the hash value and dispatch events. -- `rowid`: uses the name and value of the selected HandleKey column to create the hash value and dispatch events. +- `index-value`: uses the name and value of the selected HandleKey column to create the hash value and dispatch events. - `table`: uses the schema name of the table and the table name to create the hash value and dispatch events. `matcher`: The matching syntax of matcher is the same as the filter rule syntax. -`protocol`: For the sink of MQ type, you can specify the protocol format of the message. Currently the following protocols are supported: `canal-json`, `open-protocol`, `canal`, `avro`, and `maxwell`. +`protocol`: For the sink of MQ type, you can specify the protocol format of the message. Currently the following protocols are supported: `canal-json`, `open-protocol`, and `avro`. ### Example diff --git a/ticdc/ticdc-open-protocol.md b/ticdc/ticdc-open-protocol.md index 54ad9facf27bf..0a0247e5eae71 100644 --- a/ticdc/ticdc-open-protocol.md +++ b/ticdc/ticdc-open-protocol.md @@ -1,7 +1,6 @@ --- title: TiCDC Open Protocol summary: Learn the concept of TiCDC Open Protocol and how to use it. -aliases: ['/docs/dev/ticdc/ticdc-open-protocol/','/docs/dev/reference/tools/ticdc/open-protocol/','/docs/dev/ticdc/column-ddl-type-codes/'] --- # TiCDC Open Protocol @@ -50,7 +49,7 @@ This section introduces the formats of Row Changed Event, DDL Event, and Resolve + **Key:** - ``` + ```json { "ts":, "scm":, @@ -69,7 +68,7 @@ This section introduces the formats of Row Changed Event, DDL Event, and Resolve `Insert` event. The newly added row data is output. - ``` + ```json { "u":{ :{ @@ -90,7 +89,7 @@ This section introduces the formats of Row Changed Event, DDL Event, and Resolve `Update` event. The newly added row data ("u") and the row data before the update ("p") are output. - ``` + ```json { "u":{ :{ @@ -125,7 +124,7 @@ This section introduces the formats of Row Changed Event, DDL Event, and Resolve `Delete` event. The deleted row data is output. - ``` + ```json { "d":{ :{ @@ -156,7 +155,7 @@ This section introduces the formats of Row Changed Event, DDL Event, and Resolve + **Key:** - ``` + ```json { "ts":, "scm":, @@ -173,7 +172,7 @@ This section introduces the formats of Row Changed Event, DDL Event, and Resolve + **Value:** - ``` + ```json { "q":, "t": @@ -189,7 +188,7 @@ This section introduces the formats of Row Changed Event, DDL Event, and Resolve + **Key:** - ``` + ```json { "ts":, "t":3 @@ -241,10 +240,10 @@ COMMIT; + Log 8 is a repeated event of Log 7. Row Changed Event might be repeated, but the first Event of each version is sent orderly. ``` -5. [partition=0] [key="{\"ts\":415508878783938562,\"scm\":\"test\",\"tbl\":\"t1\",\"t\":1}"] [value="{\"u\":{\"id\":{\"t\":3,\"h\":true,\"v\":1},\"val\":{\"t\":15,\"v\":\"YWE=\"}}}"] -6. [partition=1] [key="{\"ts\":415508878783938562,\"scm\":\"test\",\"tbl\":\"t1\",\"t\":1}"] [value="{\"u\":{\"id\":{\"t\":3,\"h\":true,\"v\":2},\"val\":{\"t\":15,\"v\":\"YmI=\"}}}"] -7. [partition=0] [key="{\"ts\":415508878783938562,\"scm\":\"test\",\"tbl\":\"t1\",\"t\":1}"] [value="{\"u\":{\"id\":{\"t\":3,\"h\":true,\"v\":3},\"val\":{\"t\":15,\"v\":\"Y2M=\"}}}"] -8. [partition=0] [key="{\"ts\":415508878783938562,\"scm\":\"test\",\"tbl\":\"t1\",\"t\":1}"] [value="{\"u\":{\"id\":{\"t\":3,\"h\":true,\"v\":3},\"val\":{\"t\":15,\"v\":\"Y2M=\"}}}"] +5. [partition=0] [key="{\"ts\":415508878783938562,\"scm\":\"test\",\"tbl\":\"t1\",\"t\":1}"] [value="{\"u\":{\"id\":{\"t\":3,\"h\":true,\"v\":1},\"val\":{\"t\":15,\"v\":\"aa\"}}}"] +6. [partition=1] [key="{\"ts\":415508878783938562,\"scm\":\"test\",\"tbl\":\"t1\",\"t\":1}"] [value="{\"u\":{\"id\":{\"t\":3,\"h\":true,\"v\":2},\"val\":{\"t\":15,\"v\":\"bb\"}}}"] +7. [partition=0] [key="{\"ts\":415508878783938562,\"scm\":\"test\",\"tbl\":\"t1\",\"t\":1}"] [value="{\"u\":{\"id\":{\"t\":3,\"h\":true,\"v\":3},\"val\":{\"t\":15,\"v\":\"cc\"}}}"] +8. [partition=0] [key="{\"ts\":415508878783938562,\"scm\":\"test\",\"tbl\":\"t1\",\"t\":1}"] [value="{\"u\":{\"id\":{\"t\":3,\"h\":true,\"v\":3},\"val\":{\"t\":15,\"v\":\"cc\"}}}"] ``` Execute the following SQL statements in the upstream: @@ -275,8 +274,8 @@ COMMIT; Currently, TiCDC does not provide the standard parsing library for TiCDC Open Protocol, but the Golang version and Java version of parsing examples are provided. You can refer to the data format provided in this document and the following examples to implement the protocol parsing for consumers. -- [Golang examples](https://github.com/pingcap/tiflow/tree/master/cmd/kafka-consumer) -- [Java examples](https://github.com/pingcap/tiflow/tree/master/examples/java) +- [Golang examples](https://github.com/pingcap/tiflow/tree/release-7.5/cmd/kafka-consumer) +- [Java examples](https://github.com/pingcap/tiflow/tree/release-7.5/examples/java) ## Column type code diff --git a/ticdc/ticdc-overview.md b/ticdc/ticdc-overview.md index 805e1869d3709..e2b301ceabcc8 100644 --- a/ticdc/ticdc-overview.md +++ b/ticdc/ticdc-overview.md @@ -1,12 +1,11 @@ --- title: TiCDC Overview summary: Learn what TiCDC is, what features TiCDC provides, and how to install and deploy TiCDC. -aliases: ['/docs/dev/ticdc/ticdc-overview/','/docs/dev/reference/tools/ticdc/overview/'] --- # TiCDC Overview -[TiCDC](https://github.com/pingcap/tiflow/tree/master/cdc) is a tool used to replicate incremental data from TiDB. Specifically, TiCDC pulls TiKV change logs, sorts captured data, and exports row-based incremental data to downstream databases. +[TiCDC](https://github.com/pingcap/tiflow/tree/release-7.5/cdc) is a tool used to replicate incremental data from TiDB. Specifically, TiCDC pulls TiKV change logs, sorts captured data, and exports row-based incremental data to downstream databases. For detailed data replication capabilities, see [TiCDC Data Replication Capabilities](/ticdc/ticdc-data-replication-capabilities.md). ## Usage scenarios @@ -85,7 +84,62 @@ As shown in the architecture diagram, TiCDC supports replicating data to TiDB, M - A primary key (`PRIMARY KEY`) is a valid index. - A unique index (`UNIQUE INDEX`) is valid if every column of the index is explicitly defined as non-nullable (`NOT NULL`) and the index does not have a virtual generated column (`VIRTUAL GENERATED COLUMNS`). -- To use TiCDC in disaster recovery scenarios, you need to configure [redo log](/ticdc/ticdc-sink-to-mysql.md#eventually-consistent-replication-in-disaster-scenarios). +- To ensure eventual consistency when using TiCDC for disaster recovery, you need to configure [redo log](/ticdc/ticdc-sink-to-mysql.md#eventually-consistent-replication-in-disaster-scenarios) and ensure that the storage system where the redo log is written can be read normally when a disaster occurs in the upstream. + +## Implementation of processing data changes + +This section mainly describes how TiCDC processes data changes generated by upstream DML operations. + +For data changes generated by upstream DDL operations, TiCDC obtains the complete DDL SQL statement, converts it into the corresponding format based on the sink type of the downstream, and sends it to the downstream. This section does not elaborate on this. + +> **Note:** +> +> The logic of how TiCDC processes data changes might be adjusted in subsequent versions. + +MySQL binlog directly records all DML SQL statements executed in the upstream. Unlike MySQL, TiCDC listens to the real-time information of each Region Raft Log in the upstream TiKV, and generates data change information based on the difference between the data before and after each transaction, which corresponds to multiple SQL statements. TiCDC only guarantees that the output change events are equivalent to the changes in the upstream TiDB, and does not guarantee that it can accurately restore the SQL statements that caused the data changes in the upstream TiDB. + +Data change information includes the data change types and the data values before and after the change. The difference between the data before and after the transaction can result in three types of events: + +1. The `DELETE` event: corresponds to a `DELETE` type data change message, which contains the data before the change. + +2. The `INSERT` event: corresponds to a `PUT` type data change message, which contains the data after the change. + +3. The `UPDATE` event: corresponds to a `PUT` type data change message, which contains the data both before and after the change. + +Based on the data change information, TiCDC generates data in the appropriate formats for different downstream types, and sends the data to the downstream. For example, it generates data in formats such as Canal-JSON and Avro, and writes the data to Kafka, or converts the data back into SQL statements and sends them to the downstream MySQL or TiDB. + +Currently, when TiCDC adapts data change information for the corresponding protocol, for specific `UPDATE` events, it might split them into one `DELETE` event and one `INSERT` event. For more information, see [Split `UPDATE` events for MySQL sinks](/ticdc/ticdc-split-update-behavior.md#split-update-events-for-mysql-sinks) and [Split primary or unique key `UPDATE` events for non-MySQL sinks](/ticdc/ticdc-split-update-behavior.md#split-primary-or-unique-key-update-events-for-non-mysql-sinks). + +When the downstream is MySQL or TiDB, TiCDC cannot guarantee that the SQL statements written to the downstream are exactly the same as the SQL statements executed in the upstream. This is because TiCDC does not directly obtain the original DML statements executed in the upstream, but generates SQL statements based on the data change information. However, TiCDC ensures the consistency of the final results. + +For example, the following SQL statement is executed in the upstream: + +```sql +Create Table t1 (A int Primary Key, B int); + +BEGIN; +Insert Into t1 values(1,2); +Insert Into t1 values(2,2); +Insert Into t1 values(3,3); +Commit; + +Update t1 set b = 4 where b = 2; +``` + +TiCDC generates the following two SQL statements based on the data change information, and writes them to the downstream: + +```sql +INSERT INTO `test.t1` (`A`,`B`) VALUES (1,2),(2,2),(3,3); +UPDATE `test`.`t1` +SET `A` = CASE + WHEN `A` = 1 THEN 1 + WHEN `A` = 2 THEN 2 +END, `B` = CASE + WHEN `A` = 1 THEN 4 + WHEN `A` = 2 THEN 4 +END +WHERE `A` = 1 OR `A` = 2; +``` ## Unsupported scenarios @@ -93,5 +147,6 @@ Currently, the following scenarios are not supported: - A TiKV cluster that uses RawKV alone. - The [`CREATE SEQUENCE` DDL operation](/sql-statements/sql-statement-create-sequence.md) and the [`SEQUENCE` function](/sql-statements/sql-statement-create-sequence.md#sequence-function) in TiDB. When the upstream TiDB uses `SEQUENCE`, TiCDC ignores `SEQUENCE` DDL operations/functions performed upstream. However, DML operations using `SEQUENCE` functions can be correctly replicated. +- Currently, performing [BR data recovery](/br/backup-and-restore-overview.md) and [TiDB Lightning physical import](/tidb-lightning/tidb-lightning-physical-import-mode.md) imports on tables and databases that are being replicated by TiCDC is not supported. For more information, see [Why does replication using TiCDC stall or even stop after data restore using TiDB Lightning and BR from upstream](/ticdc/ticdc-faq.md#why-does-replication-using-ticdc-stall-or-even-stop-after-data-restore-using-tidb-lightning-physical-import-mode-and-br-from-upstream). TiCDC only partially supports scenarios involving large transactions in the upstream. For details, refer to the [TiCDC FAQ](/ticdc/ticdc-faq.md#does-ticdc-support-replicating-large-transactions-is-there-any-risk), where you can find details on whether TiCDC supports replicating large transactions and any associated risks. diff --git a/ticdc/ticdc-server-config.md b/ticdc/ticdc-server-config.md index 4425cb7b5ace4..0a0aadbb971b9 100644 --- a/ticdc/ticdc-server-config.md +++ b/ticdc/ticdc-server-config.md @@ -40,6 +40,8 @@ data-dir = "" gc-ttl = 86400 # 24 h tz = "System" cluster-id = "default" +# This parameter specifies the maximum memory threshold (in bytes) for tuning GOGC: Setting a smaller threshold increases the GC frequency. Setting a larger threshold reduces GC frequency and consumes more memory resources for the TiCDC process. Once the memory usage exceeds this threshold, GOGC Tuner stops working. The default value is 0, indicating that GOGC Tuner is disabled. +gc-tuner-memory-threshold = 0 [security] ca-path = "" diff --git a/ticdc/ticdc-sink-to-cloud-storage.md b/ticdc/ticdc-sink-to-cloud-storage.md index 3d5fa0a831d61..e90929f61b6c3 100644 --- a/ticdc/ticdc-sink-to-cloud-storage.md +++ b/ticdc/ticdc-sink-to-cloud-storage.md @@ -24,7 +24,7 @@ cdc cli changefeed create \ The output is as follows: ```shell -Info: {"upstream_id":7171388873935111376,"namespace":"default","id":"simple-replication-task","sink_uri":"s3://logbucket/storage_test?protocol=canal-json","create_time":"2022-11-29T18:52:05.566016967+08:00","start_ts":437706850431664129,"engine":"unified","config":{"case_sensitive":true,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"none","enable_partition_separator":false},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"storage":""}},"state":"normal","creator_version":"v6.5.0-master-dirty"} +Info: {"upstream_id":7171388873935111376,"namespace":"default","id":"simple-replication-task","sink_uri":"s3://logbucket/storage_test?protocol=canal-json","create_time":"{{{ .tidb-release-date }}}T18:52:05.566016967+08:00","start_ts":437706850431664129,"engine":"unified","config":{"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"none","enable_partition_separator":false},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"storage":""}},"state":"normal","creator_version":"v{{{ .tidb-version }}}"} ``` - `--server`: The address of any TiCDC server in the TiCDC cluster. @@ -59,24 +59,83 @@ For `[query_parameters]` in the URI, the following parameters can be configured: ### Configure sink URI for external storage +When storing data in a cloud storage system, you need to set different authentication parameters depending on the cloud service provider. This section describes the authentication methods when using Amazon S3, Google Cloud Storage (GCS), and Azure Blob Storage, and how to configure accounts to access the corresponding storage services. + + +
    + The following is an example configuration for Amazon S3: ```shell --sink-uri="s3://bucket/prefix?protocol=canal-json" ``` +Before replicating data, you need to set appropriate access permissions for the directory in Amazon S3: + +- Minimum permissions required by TiCDC: `s3:ListBucket`, `s3:PutObject`, and `s3:GetObject`. +- If the changefeed configuration item `sink.cloud-storage-config.flush-concurrency` is greater than 1, which means parallel uploading of single files is enabled, you need to additionally add permissions related to [ListParts](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html): + - `s3:AbortMultipartUpload` + - `s3:ListMultipartUploadParts` + - `s3:ListBucketMultipartUploads` + +If you have not created a replication data storage directory, refer to [Create a bucket](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/create-bucket.html) to create an S3 bucket in the specified region. If necessary, you can also create a folder in the bucket by referring to [Organize objects in the Amazon S3 console by using folders](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/create-folder.html). + +You can configure an account to access Amazon S3 in the following ways: + +- Method 1: Specify the access key + + If you specify an access key and a secret access key, authentication is performed according to them. In addition to specifying the key in the URI, the following methods are supported: + + - TiCDC reads the `$AWS_ACCESS_KEY_ID` and `$AWS_SECRET_ACCESS_KEY` environment variables. + - TiCDC reads the `$AWS_ACCESS_KEY` and `$AWS_SECRET_KEY` environment variables. + - TiCDC reads the shared credentials file in the path specified by the `$AWS_SHARED_CREDENTIALS_FILE` environment variable. + - TiCDC reads the shared credentials file in the `~/.aws/credentials` path. + +- Method 2: Access based on an IAM role + + Associate an [IAM role with configured permissions to access Amazon S3](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html) to the EC2 instance running the TiCDC server. After successful setup, TiCDC can directly access the corresponding directories in Amazon S3 without additional settings. + +
    +
    + The following is an example configuration for GCS: ```shell --sink-uri="gcs://bucket/prefix?protocol=canal-json" ``` +You can configure the account used to access GCS by specifying an access key. Authentication is performed according to the specified `credentials-file`. In addition to specifying the key in the URI, the following methods are supported: + +- TiCDC reads the file in the path specified by the `$GOOGLE_APPLICATION_CREDENTIALS` environment variable. +- TiCDC reads the file `~/.config/gcloud/application_default_credentials.json`. +- TiCDC obtains credentials from the metadata server when the cluster is running in GCE or GAE. + +
    +
    + The following is an example configuration for Azure Blob Storage: ```shell --sink-uri="azure://bucket/prefix?protocol=canal-json" ``` +You can configure an account to access Azure Blob Storage in the following ways: + +- Method 1: Specify a shared access signature + + If you configure `account-name` and `sas-token` in the URI, the storage account name and shared access signature token specified by this parameter are used. Because the shared access signature token has the `&` character, you need to encode it as `%26` before adding it to the URI. You can also directly encode the entire `sas-token` using percent-encoding. + +- Method 2: Specify the access key + + If you configure `account-name` and `account-key` in the URI, the storage account name and key specified by this parameter are used. In addition to specifying the key file in the URI, TiCDC can also read the key from the environment variable `$AZURE_STORAGE_KEY`. + +- Method 3: Use Azure AD to restore the backup + + Configure the environment variables `$AZURE_CLIENT_ID`, `$AZURE_TENANT_ID`, and `$AZURE_CLIENT_SECRET`. + +
    +
    + > **Tip:** > > For more information about the URI parameters of Amazon S3, GCS, and Azure Blob Storage in TiCDC, see [URI Formats of External Storage Services](/external-storage-uri.md). @@ -117,7 +176,13 @@ Data change records are saved to the following path: > **Note:** > -> The table version changes only after a DDL operation is performed on the upstream table, and the new table version is the TSO when the upstream TiDB completes the execution of the DDL. However, the change of the table version does not mean the change of the table schema. For example, adding a comment to a column does not cause the schema file content to change. +> The table version changes in the following scenarios: +> +> - The upstream TiDB performs a DDL operation on the table. +> - TiCDC schedules the table across nodes. +> - The changefeed to which the table belongs restarts. +> +> Note that the change of the table version does not mean the change of the table schema. For example, adding a comment to a column does not cause the schema file content to change. ### Index files diff --git a/ticdc/ticdc-sink-to-kafka.md b/ticdc/ticdc-sink-to-kafka.md index ba5f5d30602dc..14869d5017f92 100644 --- a/ticdc/ticdc-sink-to-kafka.md +++ b/ticdc/ticdc-sink-to-kafka.md @@ -14,14 +14,14 @@ Create a replication task by running the following command: ```shell cdc cli changefeed create \ --server=http://10.0.10.25:8300 \ - --sink-uri="kafka://127.0.0.1:9092/topic-name?protocol=canal-json&kafka-version=2.4.0&partition-num=6&max-message-bytes=67108864&replication-factor=1" \ + --sink-uri="kafka://127.0.0.1:9092,127.0.0.1:9093,127.0.0.1:9094/topic-name?protocol=canal-json&kafka-version=2.4.0&partition-num=6&max-message-bytes=67108864&replication-factor=1" \ --changefeed-id="simple-replication-task" ``` ```shell Create changefeed successfully! ID: simple-replication-task -Info: {"sink-uri":"kafka://127.0.0.1:9092/topic-name?protocol=canal-json&kafka-version=2.4.0&partition-num=6&max-message-bytes=67108864&replication-factor=1","opts":{},"create-time":"2020-03-12T22:04:08.103600025+08:00","start-ts":415241823337054209,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":".","config":{"case-sensitive":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null},"scheduler":{"type":"table-number","polling-time":-1}},"state":"normal","history":null,"error":null} +Info: {"sink-uri":"kafka://127.0.0.1:9092,127.0.0.1:9093,127.0.0.1:9094/topic-name?protocol=canal-json&kafka-version=2.4.0&partition-num=6&max-message-bytes=67108864&replication-factor=1","opts":{},"create-time":"2023-11-28T22:04:08.103600025+08:00","start-ts":415241823337054209,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":".","config":{"case-sensitive":false,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null},"scheduler":{"type":"table-number","polling-time":-1}},"state":"normal","history":null,"error":null} ``` - `--server`: The address of any TiCDC server in the TiCDC cluster. @@ -31,6 +31,19 @@ Info: {"sink-uri":"kafka://127.0.0.1:9092/topic-name?protocol=canal-json&kafka-v - `--target-ts`: Specifies the ending TSO of the changefeed. To this TSO, the TiCDC cluster stops pulling data. The default value is empty, which means that TiCDC does not automatically stop pulling data. - `--config`: Specifies the changefeed configuration file. For details, see [TiCDC Changefeed Configuration Parameters](/ticdc/ticdc-changefeed-config.md). +## Supported Kafka versions + +The following table shows the minimum supported Kafka versions for each TiCDC version: + +| TiCDC version | Minimum supported Kafka version | +|:-------------------------|:--------------------------------| +| TiCDC >= v8.1.0 | 2.1.0 | +| v7.6.0 <= TiCDC < v8.1.0 | 2.4.0 | +| v7.5.2 <= TiCDC < v8.0.0 | 2.1.0 | +| v7.5.0 <= TiCDC < v7.5.2 | 2.4.0 | +| v6.5.0 <= TiCDC < v7.5.0 | 2.1.0 | +| v6.1.0 <= TiCDC < v6.5.0 | 2.0.0 | + ## Configure sink URI for Kafka Sink URI is used to specify the connection information of the TiCDC target system. The format is as follows: @@ -49,17 +62,17 @@ The following are descriptions of sink URI parameters and values that can be con | Parameter/Parameter value | Description | | :------------------ | :------------------------------------------------------------ | -| `127.0.0.1` | The IP address of the downstream Kafka services. | -| `9092` | The port for the downstream Kafka. | +| `host` | The IP address of the downstream Kafka services. | +| `port` | The port for the downstream Kafka. | | `topic-name` | Variable. The name of the Kafka topic. | -| `kafka-version` | The version of the downstream Kafka (optional, `2.4.0` by default. Currently, the earliest supported Kafka version is `0.11.0.2` and the latest one is `3.2.0`. This value needs to be consistent with the actual version of the downstream Kafka). | +| `kafka-version` | The version of the downstream Kafka. This value needs to be consistent with the actual version of the downstream Kafka. | | `kafka-client-id` | Specifies the Kafka client ID of the replication task (optional. `TiCDC_sarama_producer_replication ID` by default). | | `partition-num` | The number of the downstream Kafka partitions (optional. The value must be **no greater than** the actual number of partitions; otherwise, the replication task cannot be created successfully. `3` by default). | -| `max-message-bytes` | The maximum size of data that is sent to Kafka broker each time (optional, `10MB` by default). From v5.0.6 and v4.0.6, the default value has changed from `64MB` and `256MB` to `10MB`. | +| `max-message-bytes` | The maximum size of data that is sent to Kafka broker each time (optional, `10MB` by default, and the maximum value is `100MB`). From v5.0.6 and v4.0.6, the default value has changed from `64MB` and `256MB` to `10MB`. | | `replication-factor` | The number of Kafka message replicas that can be saved (optional, `1` by default). This value must be greater than or equal to the value of [`min.insync.replicas`](https://kafka.apache.org/33/documentation.html#brokerconfigs_min.insync.replicas) in Kafka. | | `required-acks` | A parameter used in the `Produce` request, which notifies the broker of the number of replica acknowledgements it needs to receive before responding. Value options are `0` (`NoResponse`: no response, only `TCP ACK` is provided), `1` (`WaitForLocal`: responds only after local commits are submitted successfully), and `-1` (`WaitForAll`: responds after all replicated replicas are committed successfully. You can configure the minimum number of replicated replicas using the [`min.insync.replicas`](https://kafka.apache.org/33/documentation.html#brokerconfigs_min.insync.replicas) configuration item of the broker). (Optional, the default value is `-1`). | | `compression` | The compression algorithm used when sending messages (value options are `none`, `lz4`, `gzip`, `snappy`, and `zstd`; `none` by default). Note that the Snappy compressed file must be in the [official Snappy format](https://github.com/google/snappy). Other variants of Snappy compression are not supported.| -| `protocol` | The protocol with which messages are output to Kafka. The value options are `canal-json`, `open-protocol`, `canal`, `avro` and `maxwell`. | +| `protocol` | The protocol with which messages are output to Kafka. The value options are `canal-json`, `open-protocol`, and `avro`. | | `auto-create-topic` | Determines whether TiCDC creates the topic automatically when the `topic-name` passed in does not exist in the Kafka cluster (optional, `true` by default). | | `enable-tidb-extension` | Optional. `false` by default. When the output protocol is `canal-json`, if the value is `true`, TiCDC sends [WATERMARK events](/ticdc/ticdc-canal-json.md#watermark-event) and adds the [TiDB extension field](/ticdc/ticdc-canal-json.md#tidb-extension-field) to Kafka messages. From v6.1.0, this parameter is also applicable to the `avro` protocol. If the value is `true`, TiCDC adds [three TiDB extension fields](/ticdc/ticdc-avro-protocol.md#tidb-extension-fields) to the Kafka message. | | `max-batch-size` | New in v4.0.9. If the message protocol supports outputting multiple data changes to one Kafka message, this parameter specifies the maximum number of data changes in one Kafka message. It currently takes effect only when Kafka's `protocol` is `open-protocol` (optional, `16` by default). | @@ -90,6 +103,7 @@ The following are descriptions of sink URI parameters and values that can be con * It is recommended that you create your own Kafka Topic. At a minimum, you need to set the maximum amount of data of each message that the Topic can send to the Kafka broker, and the number of downstream Kafka partitions. When you create a changefeed, these two settings correspond to `max-message-bytes` and `partition-num`, respectively. * If you create a changefeed with a Topic that does not yet exist, TiCDC will try to create the Topic using the `partition-num` and `replication-factor` parameters. It is recommended that you specify these parameters explicitly. * In most cases, it is recommended to use the `canal-json` protocol. +* If the upstream data changes in TiCDC are infrequent, such as there might be no data changes for more than 10 minutes, it is recommended to increase the Kafka connection idle timeout in the Kafka broker configuration file. For more information, see [Why do TiCDC replication tasks to Kafka often fail with `broken pipe` errors](/ticdc/ticdc-faq.md#why-do-ticdc-replication-tasks-to-kafka-often-fail-with-broken-pipe-errors). > **Note:** > @@ -136,7 +150,18 @@ The following are examples when using Kafka SASL authentication: The minimum set of permissions required for TiCDC to function properly is as follows. - The `Create`, `Write`, and `Describe` permissions for the Topic [resource type](https://docs.confluent.io/platform/current/kafka/authorization.html#resources). - - The `DescribeConfigs` permission for the Cluster resource type. + - The `DescribeConfig` permission for the Cluster resource type. + + The usage scenarios for each permission are as follows: + + | Resource type | Type of operation | Scenario | + | :-------------| :------------- | :--------------------------------| + | Cluster | `DescribeConfig`| Gets the cluster metadata while the changefeed is running | + | Topic | `Describe` | Tries to create a topic when the changefeed starts | + | Topic | `Create` | Tries to create a topic when the changefeed starts | + | Topic | `Write` | Sends data to the topic | + + When creating or starting a changefeed, you can disable the `Describe` and `Create` permissions if the specified Kafka topic already exists. ### Integrate TiCDC with Kafka Connect (Confluent Platform) @@ -157,6 +182,28 @@ dispatchers = [ For detailed integration guide, see [Quick Start Guide on Integrating TiDB with Confluent Platform](/ticdc/integrate-confluent-using-ticdc.md). +### Integrate TiCDC with AWS Glue Schema Registry + +Starting from v7.4.0, TiCDC supports using the [AWS Glue Schema Registry](https://docs.aws.amazon.com/glue/latest/dg/schema-registry.html) as the Schema Registry when users choose the Avro protocol for data replication. The configuration example is as follows: + +```shell +./cdc cli changefeed create --server=127.0.0.1:8300 --changefeed-id="kafka-glue-test" --sink-uri="kafka://127.0.0.1:9092/topic-name?&protocol=avro&replication-factor=3" --config changefeed_glue.toml +``` + +```toml +[sink] +[sink.kafka-config.glue-schema-registry-config] +region="us-west-1" +registry-name="ticdc-test" +access-key="xxxx" +secret-access-key="xxxx" +token="xxxx" +``` + +In the above configuration, `region` and `registry-name` are required fields, while `access-key`, `secret-access-key`, and `token` are optional fields. The best practice is to set the AWS credentials as environment variables or store them in the `~/.aws/credentials` file instead of setting them in the changefeed configuration file. + +For more information, refer to the [official AWS SDK for Go V2 documentation](https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/#specifying-credentials). + ## Customize the rules for Topic and Partition dispatchers of Kafka Sink ### Matcher rules @@ -182,10 +229,10 @@ dispatchers = [ You can use topic = "xxx" to specify a Topic dispatcher and use topic expressions to implement flexible topic dispatching policies. It is recommended that the total number of topics be less than 1000. -The format of the Topic expression is `[prefix][{schema}][middle][{table}][suffix]`. +The format of the Topic expression is `[prefix]{schema}[middle][{table}][suffix]`. - `prefix`: optional. Indicates the prefix of the Topic Name. -- `[{schema}]`: optional. Used to match the schema name. +- `{schema}`: required. Used to match the schema name. Starting from v7.1.4, this parameter is optional. - `middle`: optional. Indicates the delimiter between schema name and table name. - `{table}`: optional. Used to match the table name. - `suffix`: optional. Indicates the suffix of the Topic Name. @@ -225,7 +272,7 @@ For example, for a dispatcher like `matcher = ['test.*'], topic = {schema}_{tabl You can use `partition = "xxx"` to specify a partition dispatcher. It supports five dispatchers: `default`, `index-value`, `columns`, `table`, and `ts`. The dispatcher rules are as follows: - `default`: uses the `table` dispatcher rule by default. It calculates the partition number using the schema name and table name, ensuring data from a table is sent to the same partition. As a result, the data from a single table only exists in one partition and is guaranteed to be ordered. However, this dispatcher rule limits the send throughput, and the consumption speed cannot be improved by adding consumers. -- `index-value`: calculates the partition number using either the primary key, a unique index, or an explicitly specified index, distributing table data across multiple partitions. The data from a single table is sent to multiple partitions, and the data in each partition is ordered. You can improve the consumption speed by adding consumers. +- `index-value`: calculates the partition number using either the primary key, a unique index, or an index explicitly specified by `index`, distributing table data across multiple partitions. The data from a single table is sent to multiple partitions, and the data in each partition is ordered. You can improve the consumption speed by adding consumers. - `columns`: calculates the partition number using the values of explicitly specified columns, distributing table data across multiple partitions. The data from a single table is sent to multiple partitions, and the data in each partition is ordered. You can improve the consumption speed by adding consumers. - `table`: calculates the partition number using the schema name and table name. - `ts`: calculates the partition number using the commitTs of the row change, distributing table data across multiple partitions. The data from a single table is sent to multiple partitions, and the data in each partition is ordered. You can improve the consumption speed by adding consumers. However, multiple changes of a data item might be sent to different partitions and the consumer progress of different consumers might be different, which might cause data inconsistency. Therefore, the consumer needs to sort the data from multiple partitions by commitTs before consuming. @@ -236,14 +283,14 @@ Take the following configuration of `dispatchers` as an example: [sink] dispatchers = [ {matcher = ['test.*'], partition = "index-value"}, - {matcher = ['test1.*'], partition = "index-value", index-name = "index1"}, + {matcher = ['test1.*'], partition = "index-value", index = "index1"}, {matcher = ['test2.*'], partition = "columns", columns = ["id", "a"]}, {matcher = ['test3.*'], partition = "table"}, ] ``` - Tables in the `test` database use the `index-value` dispatcher, which calculates the partition number using the value of the primary key or unique index. If a primary key exists, the primary key is used; otherwise, the shortest unique index is used. -- Tables in the `test1` table use the `index-value` dispatcher and calculate the partition number using values of all columns in the index named `index1`. If the specified index does not exist, an error is reported. Note that the index specified by `index-name` must be a unique index. +- Tables in the `test1` table use the `index-value` dispatcher and calculate the partition number using values of all columns in the index named `index1`. If the specified index does not exist, an error is reported. Note that the index specified by `index` must be a unique index. - Tables in the `test2` database use the `columns` dispatcher and calculate the partition number using the values of columns `id` and `a`. If any of the columns does not exist, an error is reported. - Tables in the `test3` database use the `table` dispatcher. - Tables in the `test4` database use the `default` dispatcher, that is the `table` dispatcher, as they do not match any of the preceding rules. @@ -403,7 +450,7 @@ The message format with handle keys only is as follows: ], "old": null, "_tidb": { // TiDB extension fields - "commitTs": 163963314122145239, + "commitTs": 429918007904436226, // A TiDB TSO timestamp "onlyHandleKey": true } } @@ -469,7 +516,7 @@ The Kafka consumer receives a message that contains the address of the large mes ], "old": null, "_tidb": { // TiDB extension fields - "commitTs": 163963314122145239, + "commitTs": 429918007904436226, // A TiDB TSO timestamp "claimCheckLocation": "s3:/claim-check-bucket/${uuid}.json" } } diff --git a/ticdc/ticdc-sink-to-mysql.md b/ticdc/ticdc-sink-to-mysql.md index 9a53ec331ab29..b79cd37f4df89 100644 --- a/ticdc/ticdc-sink-to-mysql.md +++ b/ticdc/ticdc-sink-to-mysql.md @@ -21,7 +21,7 @@ cdc cli changefeed create \ ```shell Create changefeed successfully! ID: simple-replication-task -Info: {"sink-uri":"mysql://root:123456@127.0.0.1:3306/","opts":{},"create-time":"2020-03-12T22:04:08.103600025+08:00","start-ts":415241823337054209,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":".","config":{"case-sensitive":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null},"scheduler":{"type":"table-number","polling-time":-1}},"state":"normal","history":null,"error":null} +Info: {"sink-uri":"mysql://root:123456@127.0.0.1:3306/","opts":{},"create-time":"2023-11-28T22:04:08.103600025+08:00","start-ts":415241823337054209,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":".","config":{"case-sensitive":false,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null},"scheduler":{"type":"table-number","polling-time":-1}},"state":"normal","history":null,"error":null} ``` - `--server`: The address of any TiCDC server in the TiCDC cluster. @@ -31,6 +31,11 @@ Info: {"sink-uri":"mysql://root:123456@127.0.0.1:3306/","opts":{},"create-time": - `--target-ts`: Specifies the ending TSO of the changefeed. To this TSO, the TiCDC cluster stops pulling data. The default value is empty, which means that TiCDC does not automatically stop pulling data. - `--config`: Specifies the changefeed configuration file. For details, see [TiCDC Changefeed Configuration Parameters](/ticdc/ticdc-changefeed-config.md). +> **Note:** +> +> - TiCDC only replicates incremental data. To initialize full data, use Dumpling/TiDB Lightning or BR. +> - After the full data is initialized, you need to specify the `start-ts` as the TSO when the upstream backup is performed. For example, the `pos` value in the metadata file under the Dumpling directory, or the `backupTS` value in the log output after BR completes the backup. + ## Configure sink URI for MySQL or TiDB Sink URI is used to specify the connection information of the TiCDC target system. The format is as follows: @@ -60,11 +65,13 @@ The following are descriptions of sink URI parameters and parameter values that | `worker-count` | The number of SQL statements that can be concurrently executed to the downstream (optional, `16` by default). | | `cache-prep-stmts` | Controls whether to use prepared statements when executing SQL in the downstream and enable prepared statement cache on the client side (optional, `true` by default). | | `max-txn-row` | The size of a transaction batch that can be executed to the downstream (optional, `256` by default). | +| `multi-stmt-enable` | Controls whether the SQL statements executed downstream support multiple SQL statements separated by semicolons (optional, the default value is `true`). If it is set to `false`, each SQL statement is executed as a separate transaction. If it is set to `true`, `cache-prep-stmts` does not take effect. | | `ssl-ca` | The path of the CA certificate file needed to connect to the downstream MySQL instance (optional). | | `ssl-cert` | The path of the certificate file needed to connect to the downstream MySQL instance (optional). | | `ssl-key` | The path of the certificate key file needed to connect to the downstream MySQL instance (optional). | | `time-zone` | The time zone used when connecting to the downstream MySQL instance, which is effective since v4.0.8. This is an optional parameter. If this parameter is not specified, the time zone of TiCDC service processes is used. If this parameter is set to an empty value, such as `time-zone=""`, no time zone is specified when TiCDC connects to the downstream MySQL instance and the default time zone of the downstream is used. | | `transaction-atomicity` | The atomicity level of a transaction. This is an optional parameter, with the default value of `none`. When the value is `table`, TiCDC ensures the atomicity of a single-table transaction. When the value is `none`, TiCDC splits the single-table transaction. | +| `safe-mode` | Specifies how TiCDC handles `INSERT` and `UPDATE` statements when replicating data to the downstream. When it is `true`, TiCDC converts all upstream `INSERT` statements to `REPLACE INTO` statements, and all `UPDATE` statements to `DELETE` + `REPLACE INTO` statements. Before v6.1.3, the default value of this parameter is `true`. Starting from v6.1.3, the default value is changed to `false`. When TiCDC starts, it obtains a current timestamp `ThresholdTs`. For `INSERT` and `UPDATE` statements with `CommitTs` less than `ThresholdTs`, TiCDC converts them to `REPLACE INTO` statements and `DELETE` + `REPLACE INTO` statements respectively. For `INSERT` and `UPDATE` statements with `CommitTs` greater than or equal to `ThresholdTs`, `INSERT` statements are directly replicated to the downstream, while the behavior of `UPDATE` statements follows the [TiCDC Behavior in Splitting UPDATE Events](/ticdc/ticdc-split-update-behavior.md). | To encode the database password in the sink URI using Base64, use the following command: @@ -80,7 +87,7 @@ MTIzNDU2 > **Note:** > -> When the sink URI contains special characters such as `! * ' ( ) ; : @ & = + $ , / ? % # [ ]`, you need to escape the special characters, for example, in [URI Encoder](https://meyerweb.com/eric/tools/dencoder/). +> When the sink URI contains special characters such as `! * ' ( ) ; : @ & = + $ , / ? % # [ ]`, you need to escape the special characters, for example, in [URI Encoder](https://www.urlencoder.org/). ## Eventually consistent replication in disaster scenarios diff --git a/ticdc/ticdc-sink-to-pulsar.md b/ticdc/ticdc-sink-to-pulsar.md index 5dccd7a3fff4d..53af502b73a84 100644 --- a/ticdc/ticdc-sink-to-pulsar.md +++ b/ticdc/ticdc-sink-to-pulsar.md @@ -23,7 +23,7 @@ cdc cli changefeed create \ Create changefeed successfully! ID: simple-replication-task -Info: {"upstream_id":7277814241002263370,"namespace":"default","id":"simple-replication-task","sink_uri":"pulsar://127.0.0.1:6650/consumer-test?protocol=canal-json","create_time":"2023-09-12T14:42:32.000904+08:00","start_ts":444203257406423044,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["pulsar_test.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"dispatchers":[{"matcher":["pulsar_test.*"],"partition":"","topic":"test_{schema}_{table}"}],"encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"pulsar_config":{"connection-timeout":30,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"use_file_backend":false},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"}},"state":"normal","creator_version":"v7.4.0-master-dirty","resolved_ts":444203257406423044,"checkpoint_ts":444203257406423044,"checkpoint_time":"2023-09-12 14:42:31.410"} +Info: {"upstream_id":7277814241002263370,"namespace":"default","id":"simple-replication-task","sink_uri":"pulsar://127.0.0.1:6650/consumer-test?protocol=canal-json","create_time":"{{{ .tidb-release-date }}}T14:42:32.000904+08:00","start_ts":444203257406423044,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["pulsar_test.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"dispatchers":[{"matcher":["pulsar_test.*"],"partition":"","topic":"test_{schema}_{table}"}],"encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"pulsar_config":{"connection-timeout":30,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"use_file_backend":false},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"}},"state":"normal","creator_version":"v{{{ .tidb-version }}}","resolved_ts":444203257406423044,"checkpoint_ts":444203257406423044,"checkpoint_time":"{{{ .tidb-release-date }}} 14:42:31.410"} ``` The meaning of each parameter is as follows: @@ -99,12 +99,16 @@ token-from-file="/data/pulsar/token-file.txt" basic-user-name="root" # Pulsar uses the basic account and password to authenticate the identity. Specify the password. basic-password="password" -# The certificate path for Pulsar TLS encrypted authentication. +# The certificate path on the client, which is required when Pulsar enables the mTLS authentication. auth-tls-certificate-path="/data/pulsar/certificate" -# The private key path for Pulsar TLS encrypted authentication. +# The private key path on the client, which is required when Pulsar enables the mTLS authentication. auth-tls-private-key-path="/data/pulsar/certificate.key" -# Path to trusted certificate file of the Pulsar TLS encrypted authentication. +# The path to the trusted certificate file of the Pulsar TLS authentication, which is required when Pulsar enables the mTLS authentication or TLS encrypted transmission. tls-trust-certs-file-path="/data/pulsar/tls-trust-certs-file" +# The path to the encrypted private key on the client, which is required when Pulsar enables TLS encrypted transmission. +tls-key-file-path="/data/pulsar/tls-key-file" +# The path to the encrypted certificate file on the client, which is required when Pulsar enables TLS encrypted transmission. +tls-certificate-file="/data/pulsar/tls-certificate-file" # Pulsar oauth2 issuer-url. For more information, see the Pulsar website: https://pulsar.apache.org/docs/2.10.x/client-libraries-go/#tls-encryption-and-authentication oauth2.oauth2-issuer-url="https://xxxx.auth0.com" # Pulsar oauth2 audience @@ -136,6 +140,32 @@ send-timeout=30 * You need to specify the `protocol` parameter when creating a changefeed. Currently, only the `canal-json` protocol is supported for replicating data to Pulsar. * The `pulsar-producer-cache-size` parameter indicates the number of producers cached in the Pulsar client. Because each producer in Pulsar can only correspond to one topic, TiCDC adopts the LRU method to cache producers, and the default limit is 10240. If the number of topics you need to replicate is larger than the default value, you need to increase the number. +### TLS encrypted transmission + +For v7.5.1 and later v7.5 patch versions, TiCDC supports TLS encrypted transmission for Pulsar. The configuration example is as follows: + +Sink URI: + +```shell +--sink-uri="pulsar+ssl://127.0.0.1:6651/persistent://public/default/yktest?protocol=canal-json" +``` + +Configuration: + +```toml +[sink.pulsar-config] +tls-trust-certs-file-path="/data/pulsar/tls-trust-certs-file" +``` + +If the `tlsRequireTrustedClientCertOnConnect=true` parameter is configured for your Pulsar server, you also need to configure the `tls-key-file-path` and `tls-certificate-file` parameters in the changefeed configuration file. For example: + +```toml +[sink.pulsar-config] +tls-trust-certs-file-path="/data/pulsar/tls-trust-certs-file" +tls-certificate-file="/data/pulsar/tls-certificate-file" +tls-key-file-path="/data/pulsar/tls-key-file" +``` + ### TiCDC authentication and authorization for Pulsar The following is a sample configuration when you use token authentication with Pulsar: @@ -171,32 +201,34 @@ The following is a sample configuration when you use token authentication with P token-from-file="/data/pulsar/token-file.txt" ``` -- TLS encrypted authentication +- mTLS authentication Sink URI: ```shell - --sink-uri="pulsar+ssl://127.0.0.1:6650/persistent://public/default/yktest?protocol=canal-json" + --sink-uri="pulsar+ssl://127.0.0.1:6651/persistent://public/default/yktest?protocol=canal-json" ``` Config parameters: ```toml [sink.pulsar-config] - # Certificate path of the Pulsar TLS encrypted authentication + # Certificate path of the Pulsar mTLS authentication auth-tls-certificate-path="/data/pulsar/certificate" - # Private key path of the Pulsar TLS encrypted authentication + # Private key path of the Pulsar mTLS authentication auth-tls-private-key-path="/data/pulsar/certificate.key" - # Path to trusted certificate file of the Pulsar TLS encrypted authentication + # Path to the trusted certificate file of the Pulsar mTLS authentication tls-trust-certs-file-path="/data/pulsar/tls-trust-certs-file" ``` - OAuth2 authentication + For v7.5.1 and later v7.5 patch versions, TiCDC supports the OAuth2 authentication for Pulsar. + Sink URI: ```shell - --sink-uri="pulsar+ssl://127.0.0.1:6650/persistent://public/default/yktest?protocol=canal-json" + --sink-uri="pulsar://127.0.0.1:6650/persistent://public/default/yktest?protocol=canal-json" ``` Config parameters: @@ -241,8 +273,9 @@ dispatchers = [ You can use `topic = "xxx"` to specify a topic dispatcher and use topic expressions to implement flexible topic dispatching policies. It is recommended that the total number of topics be less than 1000. -The format of a topic expression is `[prefix]{schema}[middle][{table}][suffix]`. The following are the meanings of each part: +The format of a topic expression is `[tenant_and_namespace][prefix]{schema}[middle][{table}][suffix]`. The following are the meanings of each part: +- `tenant_and_namespace`:Optional. Represents the tenant and namespace of the topic, such as `persistent://abc/def/`. If not configured, it means that the topic is in the default namespace `default` under the default tenant `public` of Pulsar. - `prefix`: Optional. Represents the prefix of the topic name. - `{schema}`: Optional. Represents the database name. - `middle`: Optional. Represents the separator between a database name and a table name. diff --git a/ticdc/ticdc-split-update-behavior.md b/ticdc/ticdc-split-update-behavior.md new file mode 100644 index 0000000000000..86be8105de940 --- /dev/null +++ b/ticdc/ticdc-split-update-behavior.md @@ -0,0 +1,148 @@ +--- +title: TiCDC Behavior in Splitting UPDATE Events +summary: Introduce the behavior changes about whether TiCDC splits `UPDATE` events, including the reasons and the impact of these changes. +--- + +# TiCDC Behavior in Splitting UPDATE Events + +## Split `UPDATE` events for MySQL sinks + +Starting from v6.5.10, v7.1.6, and v7.5.2, when using the MySQL sink, any TiCDC node that receives a request for replicating a table will fetch the current timestamp `thresholdTS` from PD before starting the replication to the downstream. Based on the value of this timestamp, TiCDC decides whether to split `UPDATE` events: + +- For transactions containing one or multiple `UPDATE` changes, if the transaction `commitTS` is less than `thresholdTS`, TiCDC splits the `UPDATE` event into a `DELETE` event and an `INSERT` event before writing them to the Sorter module. +- For `UPDATE` events with the transaction `commitTS` greater than or equal to `thresholdTS`, TiCDC does not split them. For more information, see GitHub issue [#10918](https://github.com/pingcap/tiflow/issues/10918). + +This behavior change addresses the issue of downstream data inconsistencies caused by the potentially incorrect order of `UPDATE` events received by TiCDC, which can lead to an incorrect order of split `DELETE` and `INSERT` events. + +Take the following SQL statements as an example: + +```sql +CREATE TABLE t (a INT PRIMARY KEY, b INT); +INSERT INTO t VALUES (1, 1); +INSERT INTO t VALUES (2, 2); + +BEGIN; +UPDATE t SET a = 3 WHERE a = 2; +UPDATE t SET a = 2 WHERE a = 1; +COMMIT; +``` + +In this example, the two `UPDATE` statements within the transaction have a sequential dependency on execution. The primary key `a` is changed from `2` to `3`, and then the primary key `a` is changed from `1` to `2`. After this transaction is executed, the records in the upstream database are `(2, 1)` and `(3, 2)`. + +However, the order of `UPDATE` events received by TiCDC might differ from the actual execution order of the upstream transaction. For example: + +```sql +UPDATE t SET a = 2 WHERE a = 1; +UPDATE t SET a = 3 WHERE a = 2; +``` + +- Before this behavior change, TiCDC writes these `UPDATE` events to the Sorter module and then splits them into `DELETE` and `INSERT` events. After the split, the actual execution order of these events in the downstream is as follows: + + ```sql + BEGIN; + DELETE FROM t WHERE a = 1; + REPLACE INTO t VALUES (2, 1); + DELETE FROM t WHERE a = 2; + REPLACE INTO t VALUES (3, 2); + COMMIT; + ``` + + After the downstream executes the transaction, the records in the database are `(3, 2)`, which are different from the records in the upstream database (`(2, 1)` and `(3, 2)`), indicating a data inconsistency issue. + +- After this behavior change, if the transaction `commitTS` is less than the `thresholdTS` fetched from PD when TiCDC starts replicating the corresponding table to the downstream, TiCDC splits these `UPDATE` events into `DELETE` and `INSERT` events before writing them to the Sorter module. After the sorting by the Sorter module, the actual execution order of these events in the downstream is as follows: + + ```sql + BEGIN; + DELETE FROM t WHERE a = 1; + DELETE FROM t WHERE a = 2; + REPLACE INTO t VALUES (2, 1); + REPLACE INTO t VALUES (3, 2); + COMMIT; + ``` + + After the downstream executes the transaction, the records in the downstream database are the same as those in the upstream database, which are `(2, 1)` and `(3, 2)`, ensuring data consistency. + +As you can see from the preceding example, splitting the `UPDATE` event into `DELETE` and `INSERT` events before writing them to the Sorter module ensures that all `DELETE` events are executed before `INSERT` events after the split, thereby maintaining data consistency regardless of the order of `UPDATE` events received by TiCDC. + +> **Note:** +> +> After this behavior change, when using the MySQL sink, TiCDC does not split the `UPDATE` event in most cases. Consequently, there might be primary key or unique key conflicts during changefeed runtime, causing the changefeed to restart automatically. After the restart, TiCDC will split the conflicting `UPDATE` events into `DELETE` and `INSERT` events before writing them to the Sorter module. This ensures that all events within the same transaction are correctly ordered, with all `DELETE` events preceding `INSERT` events, thus correctly completing data replication. + +## Split primary or unique key `UPDATE` events for non-MySQL sinks + +### Transactions containing a single `UPDATE` change + +Starting from v6.5.3, v7.1.1, and v7.2.0, when using a non-MySQL sink, for transactions that only contain a single update change, if the primary key or non-null unique index value is modified in an `UPDATE` event, TiCDC splits this event into `DELETE` and `INSERT` events. For more information, see GitHub issue [#9086](https://github.com/pingcap/tiflow/issues/9086). + +This change primarily addresses the issue that TiCDC only outputs the new value without the old value by default when using the CSV and AVRO protocols. Due to this issue, when the primary key or non-null unique index value changes, the consumer can only receive the new value, making it impossible to process the value before the change (for example, delete the old value). Take the following SQL as an example: + +```sql +CREATE TABLE t (a INT PRIMARY KEY, b INT); +INSERT INTO t VALUES (1, 1); +UPDATE t SET a = 2 WHERE a = 1; +``` + +In this example, the primary key `a` is updated from `1` to `2`. If the `UPDATE` event is not split, the consumer can only obtain the new value `a = 2` and cannot obtain the old value `a = 1` when using the CSV and AVRO protocols. This might cause the downstream consumer to only insert the new value `2` without deleting the old value `1`. + +### Transactions containing multiple `UPDATE` changes + +Starting from v6.5.4, v7.1.2, and v7.4.0, for transactions containing multiple changes, if the primary key or non-null unique index value is modified in the `UPDATE` event, TiCDC splits the event into `DELETE` and `INSERT` events and ensures that all events follow the sequence of `DELETE` events preceding `INSERT` events. For more information, see GitHub issue [#9430](https://github.com/pingcap/tiflow/issues/9430). + +This change primarily addresses the potential issue of primary key or unique key conflicts that consumers might encounter when writing data changes from the Kafka sink or other sinks to a relational database or performing a similar operation. This issue is caused by the potentially incorrect order of `UPDATE` events received by TiCDC. + +Take the following SQL as an example: + +```sql +CREATE TABLE t (a INT PRIMARY KEY, b INT); +INSERT INTO t VALUES (1, 1); +INSERT INTO t VALUES (2, 2); + +BEGIN; +UPDATE t SET a = 3 WHERE a = 1; +UPDATE t SET a = 1 WHERE a = 2; +UPDATE t SET a = 2 WHERE a = 3; +COMMIT; +``` + +In this example, by executing three SQL statements to swap the primary keys of two rows, TiCDC only receives two update change events, that is, changing the primary key `a` from `1` to `2` and changing the primary key `a` from `2` to `1`. If consumers directly write these two `UPDATE` events to the downstream, a primary key conflict will occur, leading to changefeed errors. + +Therefore, TiCDC splits these two events into four events, that is, deleting records `(1, 1)` and `(2, 2)` and writing records `(2, 1)` and `(1, 2)`. + +### Control whether to split primary or unique key `UPDATE` events + +Starting from v6.5.10, v7.1.6, and v7.5.3, when using a non-MySQL sink, TiCDC supports controlling whether to split primary or unique key `UPDATE` events via the `output-raw-change-event` parameter, as described in the GitHub issue [#11211]( https://github.com/pingcap/tiflow/issues/11211). The specific behavior of this parameter is as follows: + +- When you set `output-raw-change-event = false`, if the primary key or non-null unique index value is modified in an `UPDATE` event, TiCDC splits the event into `DELETE` and `INSERT` events and ensures that all events follow the sequence of `DELETE` events preceding `INSERT` events. +- When you set `output-raw-change-event = true`, TiCDC does not split `UPDATE` events, and the consumer side is responsible for dealing with the problems described in [Split primary or unique key `UPDATE` events for non-MySQL sinks](/ticdc/ticdc-split-update-behavior.md#split-primary-or-unique-key-update-events-for-non-mysql-sinks). Otherwise there might be a risk of data inconsistency. Note that when the primary key of a table is a clustered index, updates to the primary key are still split into `DELETE` and `INSERT` events in TiDB, and such behavior is not affected by the `output-raw-change-event` parameter. + +> **Note** +> +> In the following tables, UK/PK stands for primary key or unique key. + +#### Release 6.5 compatibility + +| Version | Protocol | Split UK/PK `UPDATE` events | Not split UK/PK `UPDATE` events | Comments | +| -- | -- | -- | -- | -- | +| <= v6.5.2 | ALL | ✗ | ✓ | | +| v6.5.3 / v6.5.4 | Canal/Open | ✗ | ✓ | | +| v6.5.3 | CSV/Avro | ✗ | ✗ | Split but does not sort. See [#9086](https://github.com/pingcap/tiflow/issues/9658) | +| v6.5.4 | Canal/Open | ✗ | ✗ | Only split and sort transactions that contain multiple changes | +| v6.5.5 ~ v6.5.9 | ALL | ✓ | ✗ | +| \>= v6.5.10 | ALL | ✓ (Default value: `output-raw-change-event = false`) | ✓ (Optional: `output-raw-change-event = true`) | | + +#### Release 7.1 compatibility + +| Version | Protocol | Split UK/PK `UPDATE` events | Not split UK/PK `UPDATE` events | Comments | +| -- | -- | -- | -- | -- | +| v7.1.0 | ALL | ✗ | ✓ | | +| v7.1.1 | Canal/Open | ✗ | ✓ | | +| v7.1.1 | CSV/Avro | ✗ | ✗ | Split but does not sort. See [#9086](https://github.com/pingcap/tiflow/issues/9658) | +| v7.1.2 ~ v7.1.5 | ALL | ✓ | ✗ | | +| \>= v7.1.6 | ALL | ✓ (Default value: `output-raw-change-event = false`) | ✓ (Optional: `output-raw-change-event = true`) | | + +#### Release 7.5 compatibility + +| Version | Protocol | Split UK/PK `UPDATE` events | Not split UK/PK `UPDATE` events | Comments | +| -- | -- | -- | -- | -- | +| <= v7.5.2 | ALL | ✓ | ✗ | +| \>= v7.5.3 | ALL | ✓ (Default value:`output-raw-change-event = false`) | ✓ (Optional: `output-raw-change-event = true`) | | diff --git a/ticdc/ticdc-storage-consumer-dev-guide.md b/ticdc/ticdc-storage-consumer-dev-guide.md index 18e1e01326d8a..b49c8b8600275 100644 --- a/ticdc/ticdc-storage-consumer-dev-guide.md +++ b/ticdc/ticdc-storage-consumer-dev-guide.md @@ -13,7 +13,7 @@ This document describes how to design and implement a TiDB data change consumer. TiCDC does not provide any standard way for implementing a consumer. This document provides a consumer example program written in Golang. This program can read data from the storage service and write the data to a MySQL-compatible database. You can refer to the data format and instructions provided in this example to implement a consumer on your own. -[Consumer program written in Golang](https://github.com/pingcap/tiflow/tree/master/cmd/storage-consumer) +[Consumer program written in Golang](https://github.com/pingcap/tiflow/tree/release-7.5/cmd/storage-consumer) ## Design a consumer diff --git a/ticdc/ticdc-upstream-downstream-check.md b/ticdc/ticdc-upstream-downstream-check.md index 934b2c580e740..2870c4bccdbe5 100644 --- a/ticdc/ticdc-upstream-downstream-check.md +++ b/ticdc/ticdc-upstream-downstream-check.md @@ -1,7 +1,6 @@ --- title: Upstream and Downstream Clusters Data Validation and Snapshot Read summary: Learn how to check data for TiDB upstream and downstream clusters. -aliases: ['/docs/dev/sync-diff-inspector/upstream-downstream-diff/','/docs/dev/reference/tools/sync-diff-inspector/tidb-diff/', '/tidb/dev/upstream-downstream-diff'] --- # Upstream and Downstream Clusters Data Validation and Snapshot Read @@ -36,7 +35,7 @@ sync-point-retention = "1h" > **Note:** > -> Before you perform consistent snapshot read, make sure that you have [enabled the Syncpoint feature](#enable-syncpoint). +> Before you perform consistent snapshot read, make sure that you have [enabled the Syncpoint feature](#enable-syncpoint). If multiple replication tasks use the same downstream TiDB cluster and have Syncpoint enabled, each of these tasks updates `tidb_external_ts` and `ts-map` based on their respective replication progress. In this case, you need to set up consistent snapshot read at the replication task level by reading records from the `ts-map` table. Meanwhile, you need to avoid downstream applications reading data using `tidb_enable_external_ts_read`, because multiple replication tasks might interfere with each other and result in inconsistent results. When you need to query the data from the backup cluster, you can set `SET GLOBAL|SESSION tidb_enable_external_ts_read = ON;` for the application to obtain transactionally consistent data on the backup cluster. diff --git a/ticdc/troubleshoot-ticdc.md b/ticdc/troubleshoot-ticdc.md index ccd46bd654e6a..2f52ea67a8b98 100644 --- a/ticdc/troubleshoot-ticdc.md +++ b/ticdc/troubleshoot-ticdc.md @@ -1,7 +1,6 @@ --- title: Troubleshoot TiCDC summary: Learn how to troubleshoot issues you might encounter when you use TiCDC. -aliases: ['/docs/dev/ticdc/troubleshoot-ticdc/'] --- # Troubleshoot TiCDC @@ -121,7 +120,7 @@ fetch.message.max.bytes=2147483648 ## How can I find out whether a DDL statement fails to execute in downstream during TiCDC replication? How to resume the replication? -If a DDL statement fails to execute, the replication task (changefeed) automatically stops. The checkpoint-ts is the DDL statement's finish-ts minus one. If you want TiCDC to retry executing this statement in the downstream, use `cdc cli changefeed resume` to resume the replication task. For example: +If a DDL statement fails to execute, the replication task (changefeed) automatically stops. The checkpoint-ts is the DDL statement's finish-ts. If you want TiCDC to retry executing this statement in the downstream, use `cdc cli changefeed resume` to resume the replication task. For example: {{< copyable "shell-regular" >}} @@ -131,9 +130,25 @@ cdc cli changefeed resume -c test-cf --server=http://127.0.0.1:8300 If you want to skip this DDL statement that goes wrong, set the start-ts of the changefeed to the checkpoint-ts (the timestamp at which the DDL statement goes wrong) plus one, and then run the `cdc cli changefeed create` command to create a new changefeed task. For example, if the checkpoint-ts at which the DDL statement goes wrong is `415241823337054209`, run the following commands to skip this DDL statement: -{{< copyable "shell-regular" >}} +To skip this DDL statement that goes wrong, you can configure the `ignore-txn-start-ts` parameter to skip the transactions corresponding to the specified `start-ts`. For example: + +1. Search the TiCDC log for the `apply job` field, and identify the `start-ts` of the DDLs that are taking a long time. +2. Modify the changefeed configuration. Add the above `start-ts` to the `ignore-txn-start-ts` configuration item. +3. Resume the suspended changefeed. + +> **Note:** +> +> Although setting the changefeed `start-ts` to the value of `checkpoint-ts` (at the time of the error) plus one and then recreating the task can skip the DDL statement, it can also cause TiCDC to lose the DML data change at the time of `checkpointTs+1`. Therefore, this operation is strictly prohibited in production environments. ```shell cdc cli changefeed remove --server=http://127.0.0.1:8300 --changefeed-id simple-replication-task cdc cli changefeed create --server=http://127.0.0.1:8300 --sink-uri="mysql://root:123456@127.0.0.1:3306/" --changefeed-id="simple-replication-task" --sort-engine="unified" --start-ts 415241823337054210 ``` + +## The `Kafka: client has run out of available brokers to talk to: EOF` error is reported when I use TiCDC to replicate messages to Kafka. What should I do? + +This error is typically caused by the connection failure between TiCDC and the Kafka cluster. To troubleshoot, you can check the Kafka logs and network status. One possible reason is that you did not specify the correct `kafka-version` parameter when creating the replication task, causing the Kafka client inside TiCDC to use the wrong Kafka API version when accessing the Kafka server. You can fix this issue by specifying the correct `kafka-version` parameter in the [`--sink-uri`](/ticdc/ticdc-sink-to-kafka.md#configure-sink-uri-for-kafka) configuration. For example: + +```shell +cdc cli changefeed create --server=http://127.0.0.1:8300 --sink-uri "kafka://127.0.0.1:9092/test?topic=test&protocol=open-protocol&kafka-version=2.4.0" +``` diff --git a/tidb-architecture.md b/tidb-architecture.md index 2fa322fd222e5..a0a09af07e02c 100644 --- a/tidb-architecture.md +++ b/tidb-architecture.md @@ -1,7 +1,6 @@ --- title: TiDB Architecture summary: The key architecture components of the TiDB platform -aliases: ['/docs/dev/architecture/','/tidb/dev/architecture'] --- # TiDB Architecture @@ -9,7 +8,7 @@ aliases: ['/docs/dev/architecture/','/tidb/dev/architecture'] Compared with the traditional standalone databases, TiDB has the following advantages: * Has a distributed architecture with flexible and elastic scalability. -* Fully compatible with the MySQL 5.7 protocol, common features and syntax of MySQL. To migrate your applications to TiDB, you do not need to change a single line of code in many cases. +* Fully compatible with the MySQL protocol, common features and syntax of MySQL. To migrate your applications to TiDB, you do not need to change a single line of code in many cases. * Supports high availability with automatic failover when a minority of replicas fail; transparent to applications. * Supports ACID transactions, suitable for scenarios requiring strong consistency such as bank transfer. diff --git a/tidb-binlog-deployment-topology.md b/tidb-binlog-deployment-topology.md index 3f991426c7b2b..e432fa7c1d501 100644 --- a/tidb-binlog-deployment-topology.md +++ b/tidb-binlog-deployment-topology.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Deployment Topology summary: Learn the deployment topology of TiDB Binlog based on the minimal TiDB topology. -aliases: ['/docs/dev/tidb-binlog-deployment-topology/'] --- # TiDB Binlog Deployment Topology diff --git a/tidb-binlog/bidirectional-replication-between-tidb-clusters.md b/tidb-binlog/bidirectional-replication-between-tidb-clusters.md index dc3a478e7a7b9..f471675b70a65 100644 --- a/tidb-binlog/bidirectional-replication-between-tidb-clusters.md +++ b/tidb-binlog/bidirectional-replication-between-tidb-clusters.md @@ -1,14 +1,16 @@ --- title: Bidirectional Replication Between TiDB Clusters summary: Learn how to perform the bidirectional replication between TiDB clusters. -aliases: ['/docs/dev/tidb-binlog/bidirectional-replication-between-tidb-clusters/','/docs/dev/reference/tidb-binlog/bidirectional-replication/'] --- # Bidirectional Replication between TiDB Clusters > **Warning:** > -> Currently, bidirectional replication is still an experimental feature. It is **NOT** recommended to use it in the production environment. +> - Currently, bidirectional replication is still an experimental feature. It is **NOT** recommended to use it in the production environment. +> - TiDB Binlog is not compatible with some features introduced in TiDB v5.0 and they cannot be used together. For details, see [Notes](/tidb-binlog/tidb-binlog-overview.md#notes). +> - Starting from TiDB v7.5.0, technical support for the data replication feature of TiDB Binlog is no longer provided. It is strongly recommended to use [TiCDC](/ticdc/ticdc-overview.md) as an alternative solution for data replication. +> - Although TiDB v7.5.0 still supports the real-time backup and restoration feature of TiDB Binlog, this component will be completely deprecated in future versions. It is recommended to use [PITR](/br/br-pitr-guide.md) as an alternative solution for data recovery. This document describes the bidirectional replication between two TiDB clusters, how the replication works, how to enable it, and how to replicate DDL operations. diff --git a/tidb-binlog/binlog-consumer-client.md b/tidb-binlog/binlog-consumer-client.md index e77f63314cc24..44f199704a1d6 100644 --- a/tidb-binlog/binlog-consumer-client.md +++ b/tidb-binlog/binlog-consumer-client.md @@ -1,7 +1,6 @@ --- title: Binlog Consumer Client User Guide summary: Use Binlog Consumer Client to consume TiDB secondary binlog data from Kafka and output the data in a specific format. -aliases: ['/tidb/dev/binlog-slave-client','/docs/dev/tidb-binlog/binlog-slave-client/','/docs/dev/reference/tidb-binlog/binlog-slave-client/','/docs/dev/reference/tools/tidb-binlog/binlog-slave-client/'] --- # Binlog Consumer Client User Guide @@ -118,11 +117,11 @@ message Binlog { } ``` -For the definition of the data format, see [`secondary_binlog.proto`](https://github.com/pingcap/tidb/blob/master/pkg/tidb-binlog/proto/proto/secondary_binlog.proto) +For the definition of the data format, see [`secondary_binlog.proto`](https://github.com/pingcap/tidb/blob/release-7.5/pkg/tidb-binlog/proto/proto/secondary_binlog.proto) ### Driver -The [TiDB-Tools](https://github.com/pingcap/tidb-tools/) project provides [Driver](https://github.com/pingcap/tidb/tree/master/pkg/tidb-binlog/driver), which is used to read the binlog data in Kafka. It has the following features: +The [TiDB-Tools](https://github.com/pingcap/tidb-tools/) project provides [Driver](https://github.com/pingcap/tidb/tree/release-7.5/pkg/tidb-binlog/driver), which is used to read the binlog data in Kafka. It has the following features: * Read the Kafka data. * Locate the binlog stored in Kafka based on `commit ts`. diff --git a/tidb-binlog/binlog-control.md b/tidb-binlog/binlog-control.md index 94b65503a82ae..fbf457948c475 100644 --- a/tidb-binlog/binlog-control.md +++ b/tidb-binlog/binlog-control.md @@ -1,12 +1,11 @@ --- title: binlogctl summary: Learns how to use `binlogctl`. -aliases: ['/docs/dev/tidb-binlog/binlog-control/'] --- # binlogctl -[Binlog Control](https://github.com/pingcap/tidb-binlog/tree/master/binlogctl) (`binlogctl` for short) is a command line tool for TiDB Binlog. You can use `binlogctl` to manage TiDB Binlog clusters. +[Binlog Control](https://github.com/pingcap/tidb-binlog/tree/release-7.5/binlogctl) (`binlogctl` for short) is a command line tool for TiDB Binlog. You can use `binlogctl` to manage TiDB Binlog clusters. You can use `binlogctl` to: diff --git a/tidb-binlog/deploy-tidb-binlog.md b/tidb-binlog/deploy-tidb-binlog.md index 10699145008c0..f40e240bba26c 100644 --- a/tidb-binlog/deploy-tidb-binlog.md +++ b/tidb-binlog/deploy-tidb-binlog.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Cluster Deployment summary: Learn how to deploy TiDB Binlog cluster. -aliases: ['/docs/dev/tidb-binlog/deploy-tidb-binlog/','/docs/dev/reference/tidb-binlog/deploy/','/docs/dev/how-to/deploy/tidb-binlog/'] --- # TiDB Binlog Cluster Deployment diff --git a/tidb-binlog/get-started-with-tidb-binlog.md b/tidb-binlog/get-started-with-tidb-binlog.md index 8e0fd9320c06e..c000e0cac8c8c 100644 --- a/tidb-binlog/get-started-with-tidb-binlog.md +++ b/tidb-binlog/get-started-with-tidb-binlog.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Tutorial summary: Learn to deploy TiDB Binlog with a simple TiDB cluster. -aliases: ['/docs/dev/get-started-with-tidb-binlog/','/docs/dev/how-to/get-started/tidb-binlog/'] --- # TiDB Binlog Tutorial @@ -43,14 +42,14 @@ sudo yum install -y mariadb-server ``` ```bash -curl -L https://download.pingcap.org/tidb-community-server-v7.4.0-linux-amd64.tar.gz | tar xzf - +curl -L https://download.pingcap.com/tidb-community-server-v{{{ .tidb-version }}}-linux-amd64.tar.gz | tar xzf - cd tidb-latest-linux-amd64 ``` Expected output: ``` -[kolbe@localhost ~]$ curl -LO https://download.pingcap.org/tidb-latest-linux-amd64.tar.gz | tar xzf - +[kolbe@localhost ~]$ curl -LO https://download.pingcap.com/tidb-latest-linux-amd64.tar.gz | tar xzf - % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 100 368M 100 368M 0 0 8394k 0 0:00:44 0:00:44 --:--:-- 11.1M diff --git a/tidb-binlog/handle-tidb-binlog-errors.md b/tidb-binlog/handle-tidb-binlog-errors.md index 5ba106db49415..b10262abcdd2f 100644 --- a/tidb-binlog/handle-tidb-binlog-errors.md +++ b/tidb-binlog/handle-tidb-binlog-errors.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Error Handling summary: Learn how to handle TiDB Binlog errors. -aliases: ['/docs/dev/tidb-binlog/handle-tidb-binlog-errors/','/docs/dev/reference/tidb-binlog/troubleshoot/error-handling/'] --- # TiDB Binlog Error Handling diff --git a/tidb-binlog/maintain-tidb-binlog-cluster.md b/tidb-binlog/maintain-tidb-binlog-cluster.md index 5690e7dcf16eb..c846b5217a959 100644 --- a/tidb-binlog/maintain-tidb-binlog-cluster.md +++ b/tidb-binlog/maintain-tidb-binlog-cluster.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Cluster Operations summary: Learn how to operate the cluster version of TiDB Binlog. -aliases: ['/docs/dev/tidb-binlog/maintain-tidb-binlog-cluster/','/docs/dev/reference/tidb-binlog/maintain/','/docs/dev/how-to/maintain/tidb-binlog/','/docs/dev/reference/tools/tidb-binlog/maintain/'] --- # TiDB Binlog Cluster Operations @@ -46,7 +45,7 @@ For how to pause, close, check, and modify the state of Drainer, see the [binlog ## Use `binlogctl` to manage Pump/Drainer -[`binlogctl`](https://github.com/pingcap/tidb-binlog/tree/master/binlogctl) is an operations tool for TiDB Binlog with the following features: +[`binlogctl`](https://github.com/pingcap/tidb-binlog/tree/release-7.5/binlogctl) is an operations tool for TiDB Binlog with the following features: * Checking the state of Pump or Drainer * Pausing or closing Pump or Drainer diff --git a/tidb-binlog/monitor-tidb-binlog-cluster.md b/tidb-binlog/monitor-tidb-binlog-cluster.md index a5bf64f19482a..19831474c3b89 100644 --- a/tidb-binlog/monitor-tidb-binlog-cluster.md +++ b/tidb-binlog/monitor-tidb-binlog-cluster.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Monitoring summary: Learn how to monitor the cluster version of TiDB Binlog. -aliases: ['/docs/dev/tidb-binlog/monitor-tidb-binlog-cluster/','/docs/dev/reference/tidb-binlog/monitor/','/docs/dev/how-to/monitor/tidb-binlog/'] --- # TiDB Binlog Monitoring diff --git a/tidb-binlog/tidb-binlog-configuration-file.md b/tidb-binlog/tidb-binlog-configuration-file.md index d825012ca9b4c..b2a1ecebb496f 100644 --- a/tidb-binlog/tidb-binlog-configuration-file.md +++ b/tidb-binlog/tidb-binlog-configuration-file.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Configuration File summary: Learn the configuration items of TiDB Binlog. -aliases: ['/docs/dev/tidb-binlog/tidb-binlog-configuration-file/','/docs/dev/reference/tidb-binlog/config/'] --- # TiDB Binlog Configuration File @@ -10,7 +9,7 @@ This document introduces the configuration items of TiDB Binlog. ## Pump -This section introduces the configuration items of Pump. For the example of a complete Pump configuration file, see [Pump Configuration](https://github.com/pingcap/tidb-binlog/blob/master/cmd/pump/pump.toml). +This section introduces the configuration items of Pump. For the example of a complete Pump configuration file, see [Pump Configuration](https://github.com/pingcap/tidb-binlog/blob/release-7.5/cmd/pump/pump.toml). ### addr @@ -129,7 +128,7 @@ For the detailed description of the above items, see [GoLevelDB Document](https: ## Drainer -This section introduces the configuration items of Drainer. For the example of a complete Drainer configuration file, see [Drainer Configuration](https://github.com/pingcap/tidb-binlog/blob/master/cmd/drainer/drainer.toml) +This section introduces the configuration items of Drainer. For the example of a complete Drainer configuration file, see [Drainer Configuration](https://github.com/pingcap/tidb-binlog/blob/release-7.5/cmd/drainer/drainer.toml) ### addr @@ -306,6 +305,14 @@ If the safe mode is enabled, Drainer modifies the replication updates in the fol Default value: `false` +#### load-schema-snapshot + +- Specifies how Drainer loads table information. +- When you set it to `false`, Drainer replays all DDL operations from history to derive the table schema for each table at a specific schema version. This approach requires processing all DDL changes from the initial state to the target schema version, which might involve significant data processing and replaying. +- When you set it to `true`, Drainer directly reads the table information at the checkpoint TS. Becasue it directly reads the table information at a specific point in time, this method is usually more efficient. However, it is subject to the GC mechanism, because GC might delete older data versions. If the checkpoint TS is too old, the corresponding table information might have been deleted by GC, making it impossible to read directly. +- When configuring Drainer, choose whether to read the table information at the checkpoint TS based on actual needs. If data integrity and consistency are priorities and handling a large number of DDL changes is acceptable, it is recommended to set it to `false`. If efficiency and performance are more important, and the checkpoint TS is guaranteed to be after the GC safe point, it is recommended to set it to `true`. +- Default value: `false` + ### syncer.to The `syncer.to` section introduces different types of downstream configuration items according to configuration types. @@ -350,4 +357,4 @@ When the downstream is Kafka, the valid configuration items are as follows: * `host` * `user` * `password` -* `port` \ No newline at end of file +* `port` diff --git a/tidb-binlog/tidb-binlog-faq.md b/tidb-binlog/tidb-binlog-faq.md index 5218386ca2e54..6de377e22a005 100644 --- a/tidb-binlog/tidb-binlog-faq.md +++ b/tidb-binlog/tidb-binlog-faq.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog FAQs summary: Learn about the frequently asked questions (FAQs) and answers about TiDB Binlog. -aliases: ['/docs/dev/tidb-binlog/tidb-binlog-faq/','/docs/dev/reference/tidb-binlog/faq/','/docs/dev/reference/tools/tidb-binlog/faq/'] --- # TiDB Binlog FAQs diff --git a/tidb-binlog/tidb-binlog-glossary.md b/tidb-binlog/tidb-binlog-glossary.md index 4c9ae8b8d25e4..2f6351209d111 100644 --- a/tidb-binlog/tidb-binlog-glossary.md +++ b/tidb-binlog/tidb-binlog-glossary.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Glossary summary: Learn the terms used in TiDB Binlog. -aliases: ['/docs/dev/tidb-binlog/tidb-binlog-glossary/','/docs/dev/reference/tidb-binlog/glossary/'] --- # TiDB Binlog Glossary diff --git a/tidb-binlog/tidb-binlog-overview.md b/tidb-binlog/tidb-binlog-overview.md index 3af9fe6b163e8..5bd31cacffa81 100644 --- a/tidb-binlog/tidb-binlog-overview.md +++ b/tidb-binlog/tidb-binlog-overview.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Overview summary: Learn overview of the cluster version of TiDB Binlog. -aliases: ['/docs/dev/tidb-binlog/tidb-binlog-overview/','/docs/dev/reference/tidb-binlog/overview/','/docs/dev/reference/tidb-binlog-overview/','/docs/dev/reference/tools/tidb-binlog/overview/'] --- # TiDB Binlog Cluster Overview @@ -17,7 +16,9 @@ TiDB Binlog has the following features: > **Note:** > -> TiDB Binlog is not compatible with some features introduced in TiDB v5.0 and they cannot be used together. For details, see [Notes](#notes). It is recommended to use [TiCDC](/ticdc/ticdc-overview.md) instead of TiDB Binlog. +> - TiDB Binlog is not compatible with some features introduced in TiDB v5.0 and they cannot be used together. For details, see [Notes](#notes). +> - Starting from TiDB v7.5.0, technical support for the data replication feature of TiDB Binlog is no longer provided. It is strongly recommended to use [TiCDC](/ticdc/ticdc-overview.md) as an alternative solution for data replication. +> - Although TiDB v7.5.0 still supports the real-time backup and restoration feature of TiDB Binlog, this component will be completely deprecated in future versions. It is recommended to use [PITR](/br/br-pitr-guide.md) as an alternative solution for data recovery. ## TiDB Binlog architecture @@ -29,15 +30,15 @@ The TiDB Binlog cluster is composed of Pump and Drainer. ### Pump -[Pump](https://github.com/pingcap/tidb-binlog/blob/master/pump) is used to record the binlogs generated in TiDB, sort the binlogs based on the commit time of the transaction, and send binlogs to Drainer for consumption. +[Pump](https://github.com/pingcap/tidb-binlog/blob/release-7.5/pump) is used to record the binlogs generated in TiDB, sort the binlogs based on the commit time of the transaction, and send binlogs to Drainer for consumption. ### Drainer -[Drainer](https://github.com/pingcap/tidb-binlog/tree/master/drainer) collects and merges binlogs from each Pump, converts the binlog to SQL or data of a specific format, and replicates the data to a specific downstream platform. +[Drainer](https://github.com/pingcap/tidb-binlog/tree/release-7.5/drainer) collects and merges binlogs from each Pump, converts the binlog to SQL or data of a specific format, and replicates the data to a specific downstream platform. ### `binlogctl` guide -[`binlogctl`](https://github.com/pingcap/tidb-binlog/tree/master/binlogctl) is an operations tool for TiDB Binlog with the following features: +[`binlogctl`](https://github.com/pingcap/tidb-binlog/tree/release-7.5/binlogctl) is an operations tool for TiDB Binlog with the following features: * Obtaining the current `tso` of TiDB cluster * Checking the Pump/Drainer state @@ -67,7 +68,7 @@ The TiDB Binlog cluster is composed of Pump and Drainer. * Drainer supports replicating binlogs to MySQL, TiDB, Kafka or local files. If you need to replicate binlogs to other Drainer unsuppored destinations, you can set Drainer to replicate the binlog to Kafka and read the data in Kafka for customized processing according to binlog consumer protocol. See [Binlog Consumer Client User Guide](/tidb-binlog/binlog-consumer-client.md). -* To use TiDB Binlog for recovering incremental data, set the config `db-type` to `file` (local files in the proto buffer format). Drainer converts the binlog to data in the specified [proto buffer format](https://github.com/pingcap/tidb-binlog/blob/master/proto/pb_binlog.proto) and writes the data to local files. In this way, you can use [Reparo](/tidb-binlog/tidb-binlog-reparo.md) to recover data incrementally. +* To use TiDB Binlog for recovering incremental data, set the config `db-type` to `file` (local files in the proto buffer format). Drainer converts the binlog to data in the specified [proto buffer format](https://github.com/pingcap/tidb-binlog/blob/release-7.5/proto/pb_binlog.proto) and writes the data to local files. In this way, you can use [Reparo](/tidb-binlog/tidb-binlog-reparo.md) to recover data incrementally. Pay attention to the value of `db-type`: diff --git a/tidb-binlog/tidb-binlog-relay-log.md b/tidb-binlog/tidb-binlog-relay-log.md index 68229df5b7463..752699b1f9c27 100644 --- a/tidb-binlog/tidb-binlog-relay-log.md +++ b/tidb-binlog/tidb-binlog-relay-log.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Relay Log summary: Learn how to use relay log to maintain data consistency in extreme cases. -aliases: ['/docs/dev/tidb-binlog/tidb-binlog-relay-log/','/docs/dev/reference/tidb-binlog/relay-log/'] --- # TiDB Binlog Relay Log diff --git a/tidb-binlog/tidb-binlog-reparo.md b/tidb-binlog/tidb-binlog-reparo.md index c4f80a5015f5b..d6b8d02b505b1 100644 --- a/tidb-binlog/tidb-binlog-reparo.md +++ b/tidb-binlog/tidb-binlog-reparo.md @@ -1,7 +1,6 @@ --- title: Reparo User Guide summary: Learn to use Reparo. -aliases: ['/docs/dev/tidb-binlog/tidb-binlog-reparo/','/docs/dev/reference/tidb-binlog/reparo/'] --- # Reparo User Guide diff --git a/tidb-binlog/troubleshoot-tidb-binlog.md b/tidb-binlog/troubleshoot-tidb-binlog.md index decd1c675c2c3..b3ab2fce68e8b 100644 --- a/tidb-binlog/troubleshoot-tidb-binlog.md +++ b/tidb-binlog/troubleshoot-tidb-binlog.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Troubleshooting summary: Learn the troubleshooting process of TiDB Binlog. -aliases: ['/docs/dev/tidb-binlog/troubleshoot-tidb-binlog/','/docs/dev/reference/tidb-binlog/troubleshoot/binlog/','/docs/dev/how-to/troubleshoot/tidb-binlog/'] --- # TiDB Binlog Troubleshooting diff --git a/tidb-binlog/upgrade-tidb-binlog.md b/tidb-binlog/upgrade-tidb-binlog.md index 9ba74a465cb3e..046fcff9ae789 100644 --- a/tidb-binlog/upgrade-tidb-binlog.md +++ b/tidb-binlog/upgrade-tidb-binlog.md @@ -1,13 +1,18 @@ --- title: Upgrade TiDB Binlog summary: Learn how to upgrade TiDB Binlog to the latest cluster version. -aliases: ['/docs/dev/tidb-binlog/upgrade-tidb-binlog/','/docs/dev/reference/tidb-binlog/upgrade/','/docs/dev/how-to/upgrade/tidb-binlog/'] --- # Upgrade TiDB Binlog This document introduces how to upgrade TiDB Binlog that is deployed manually to the latest [cluster](/tidb-binlog/tidb-binlog-overview.md) version. There is also a section on how to upgrade TiDB Binlog from an earlier incompatible version (Kafka/Local version) to the latest version. +> **Note:** +> +> - TiDB Binlog is not compatible with some features introduced in TiDB v5.0 and they cannot be used together. For details, see [Notes](/tidb-binlog/tidb-binlog-overview.md#notes). +> - Starting from TiDB v7.5.0, technical support for the data replication feature of TiDB Binlog is no longer provided. It is strongly recommended to use [TiCDC](/ticdc/ticdc-overview.md) as an alternative solution for data replication. +> - Although TiDB v7.5.0 still supports the real-time backup and restoration feature of TiDB Binlog, this component will be completely deprecated in future versions. It is recommended to use [PITR](/br/br-pitr-guide.md) as an alternative solution for data recovery. + ## Upgrade TiDB Binlog deployed manually Follow the steps in this section if you deploy TiDB Binlog manually. diff --git a/tidb-cloud/_index.md b/tidb-cloud/_index.md index fe569f7a24f58..689593572eac3 100644 --- a/tidb-cloud/_index.md +++ b/tidb-cloud/_index.md @@ -3,6 +3,7 @@ title: TiDB Cloud Documentation aliases: ['/tidbcloud/privacy-policy', '/tidbcloud/terms-of-service', '/tidbcloud/service-level-agreement'] hide_sidebar: true hide_commit: true +summary: TiDB Cloud is a fully-managed Database-as-a-Service (DBaaS) that brings everything great about TiDB to your cloud. It offers guides, samples, and references for learning, trying, developing, maintaining, migrating, monitoring, tuning, securing, billing, integrating, and referencing. --- @@ -21,6 +22,8 @@ hide_commit: true [Try Out TiDB Cloud](https://docs.pingcap.com/tidbcloud/tidb-cloud-quickstart) +[Try Out TiDB + AI](https://docs.pingcap.com/tidbcloud/vector-search-get-started-using-python) + [Try Out HTAP](https://docs.pingcap.com/tidbcloud/tidb-cloud-htap-quickstart) [Proof of Concept](https://docs.pingcap.com/tidbcloud/tidb-cloud-poc) @@ -73,6 +76,8 @@ hide_commit: true [From Apache Parquet Files](https://docs.pingcap.com/tidbcloud/import-csv-files) +[With MySQL CLI](https://docs.pingcap.com/tidbcloud/import-with-mysql-cli) + @@ -109,13 +114,13 @@ hide_commit: true [Manage project access](https://docs.pingcap.com/tidbcloud/manage-user-access#manage-project-access) -[Configure Security Settings](https://docs.pingcap.com/tidbcloud/configure-security-settings) +[Configure Password Settings](https://docs.pingcap.com/tidbcloud/configure-security-settings) -[Pricing](https://en.pingcap.com/tidb-cloud-pricing/) +[Pricing](https://www.pingcap.com/pricing/) [Invoices](https://docs.pingcap.com/tidbcloud/tidb-cloud-billing#invoices) diff --git a/tidb-cloud/api-overview.md b/tidb-cloud/api-overview.md index 19810b1e97ff4..176bf2cdede7b 100644 --- a/tidb-cloud/api-overview.md +++ b/tidb-cloud/api-overview.md @@ -7,20 +7,26 @@ summary: Learn about what is TiDB Cloud API, its features, and how to use API to > **Note:** > -> [TiDB Cloud API](https://docs.pingcap.com/tidbcloud/api/v1beta) is in beta. +> TiDB Cloud API is in beta. -The TiDB Cloud API is a [REST interface](https://en.wikipedia.org/wiki/Representational_state_transfer) that provides you with programmatic access to manage administrative objects within TiDB Cloud. Through this API, you can automatically and efficiently manage resources such as Projects, Clusters, Backups, Restores, and Imports. +The TiDB Cloud API is a [REST interface](https://en.wikipedia.org/wiki/Representational_state_transfer) that provides you with programmatic access to manage administrative objects within TiDB Cloud. Through this API, you can automatically and efficiently manage resources such as Projects, Clusters, Backups, Restores, Imports, Billings, and resources in the [Data Service](/tidb-cloud/data-service-overview.md). The API has the following features: - **JSON entities.** All entities are expressed in JSON. - **HTTPS-only.** You can only access the API via HTTPS, ensuring all the data sent over the network is encrypted with TLS. -- **Key-based access and digest authentication.** Before you access TiDB Cloud API, you must generate an API key. All requests are authenticated through [HTTP Digest Authentication](https://en.wikipedia.org/wiki/Digest_access_authentication), ensuring the API key is never sent over the network. +- **Key-based access and digest authentication.** Before you access TiDB Cloud API, you must generate an API key, refer to [API Key Management](https://docs.pingcap.com/tidbcloud/api/v1beta#section/Authentication/API-key-management). All requests are authenticated through [HTTP Digest Authentication](https://en.wikipedia.org/wiki/Digest_access_authentication), ensuring the API key is never sent over the network. -To start using TiDB Cloud API, refer to the following resources in [TiDB Cloud API Documentation](https://docs.pingcap.com/tidbcloud/api/v1beta): +To start using TiDB Cloud API, refer to the following resources in TiDB Cloud API Documentation: - [Get Started](https://docs.pingcap.com/tidbcloud/api/v1beta#section/Get-Started) - [Authentication](https://docs.pingcap.com/tidbcloud/api/v1beta#section/Authentication) - [Rate Limiting](https://docs.pingcap.com/tidbcloud/api/v1beta#section/Rate-Limiting) -- [API Full References](https://docs.pingcap.com/tidbcloud/api/v1beta#tag/Project) +- API Full References + - v1beta1 + - [Billing](https://docs.pingcap.com/tidbcloud/api/v1beta1/billing) + - [Data Service](https://docs.pingcap.com/tidbcloud/api/v1beta1/dataservice) + - [IAM](https://docs.pingcap.com/tidbcloud/api/v1beta1/iam) + - [MSP (Deprecated)](https://docs.pingcap.com/tidbcloud/api/v1beta1/msp) + - [v1beta](https://docs.pingcap.com/tidbcloud/api/v1beta#tag/Project) - [Changelog](https://docs.pingcap.com/tidbcloud/api/v1beta#section/API-Changelog) diff --git a/tidb-cloud/backup-and-restore-serverless.md b/tidb-cloud/backup-and-restore-serverless.md index ba0a494131f86..9cea220a344a8 100644 --- a/tidb-cloud/backup-and-restore-serverless.md +++ b/tidb-cloud/backup-and-restore-serverless.md @@ -1,33 +1,33 @@ --- -title: Back Up and Restore TiDB Serverless Data -summary: Learn how to back up and restore your TiDB Serverless cluster. +title: Back Up and Restore TiDB Cloud Serverless Data +summary: Learn how to back up and restore your TiDB Cloud Serverless cluster. aliases: ['/tidbcloud/restore-deleted-tidb-cluster'] --- -# Back Up and Restore TiDB Serverless Data +# Back Up and Restore TiDB Cloud Serverless Data -This document describes how to back up and restore your TiDB Serverless cluster data on TiDB Cloud. +This document describes how to back up and restore your TiDB Cloud Serverless cluster data on TiDB Cloud. > **Tip:** > -> To learn how to back up and restore TiDB Dedicated cluster data, see [Back Up and Restore TiDB Dedicated Data](/tidb-cloud/backup-and-restore.md). +> To learn how to back up and restore TiDB Cloud Dedicated cluster data, see [Back Up and Restore TiDB Cloud Dedicated Data](/tidb-cloud/backup-and-restore.md). ## Limitations -- It is important to note that TiDB Serverless clusters only support in-place restoring from backups. When a restore is performed, tables in the `mysql` schema are also impacted. Hence, any changes made to user credentials and permissions or system variables will be rolled back to the state when the backup was taken. +- It is important to note that TiDB Cloud Serverless clusters only support in-place restoring from backups. When a restore is performed, tables in the `mysql` schema are also impacted. Hence, any changes made to user credentials and permissions or system variables will be rolled back to the state when the backup was taken. - Manual backup is not yet supported. - The cluster will be unavailable during the restore process, and existing connections will be terminated. You can establish new connections once the restore is complete. - If any TiFlash replica is enabled, the replica will be unavailable for a while after the restore because data needs to be rebuilt in TiFlash. ## Backup -Automatic backups are scheduled for your TiDB Serverless clusters according to the backup setting, which can reduce your loss in extreme disaster situations. +Automatic backups are scheduled for your TiDB Cloud Serverless clusters according to the backup setting, which can reduce your loss in extreme disaster situations. ### Automatic backup -By the automatic backup, you can back up the TiDB Serverless cluster data every day at the backup time you have set. To set the backup time, perform the following steps: +By the automatic backup, you can back up the TiDB Cloud Serverless cluster data every day at the backup time you have set. To set the backup time, perform the following steps: -1. Navigate to the **Backup** page of a TiDB Serverless cluster. +1. Navigate to the **Backup** page of a TiDB Cloud Serverless cluster. 2. Click **Backup Settings**. This will open the **Backup Settings** window, where you can configure the automatic backup settings according to your requirements. @@ -51,7 +51,7 @@ To delete an existing backup file, perform the following steps: ## Restore -TiDB Serverless only supports in-place restoration. To restore your TiDB Serverless cluster from a backup, follow these steps: +TiDB Cloud Serverless only supports in-place restoration. To restore your TiDB Cloud Serverless cluster from a backup, follow these steps: 1. Navigate to the **Backup** page of a cluster. diff --git a/tidb-cloud/backup-and-restore.md b/tidb-cloud/backup-and-restore.md index be11dd35b9a48..f2413aba3e336 100644 --- a/tidb-cloud/backup-and-restore.md +++ b/tidb-cloud/backup-and-restore.md @@ -1,90 +1,165 @@ --- -title: Back Up and Restore TiDB Dedicated Data -summary: Learn how to back up and restore your TiDB Dedicated cluster. +title: Back Up and Restore TiDB Cloud Dedicated Data +summary: Learn how to back up and restore your TiDB Cloud Dedicated cluster. aliases: ['/tidbcloud/restore-deleted-tidb-cluster'] --- -# Back Up and Restore TiDB Dedicated Data +# Back Up and Restore TiDB Cloud Dedicated Data -This document describes how to back up and restore your TiDB Dedicated cluster data on TiDB Cloud. +This document describes how to back up and restore your TiDB Cloud Dedicated cluster data on TiDB Cloud. TiDB Cloud Dedicated supports automatic backup and manual backup. You can also restore backup data to a new cluster or restore a deleted cluster from the recycle bin. -> **Tip:** +> **Tip** > -> To learn how to back up and restore TiDB Serverless cluster data, see [Back Up and Restore TiDB Serverless Data](/tidb-cloud/backup-and-restore-serverless.md). +> To learn how to back up and restore TiDB Cloud Serverless cluster data, see [Back Up and Restore TiDB Cloud Serverless Data](/tidb-cloud/backup-and-restore-serverless.md). ## Limitations -- TiDB Cloud does not support restoring tables in the `mysql` schema, including user permissions and system variables. -- If you turn on and off PITR (Point-in-time Recovery) multiple times, you can only choose a time point within the recoverable range after the most recent PITR is enabled. The earlier recoverable range is not accessible. +- For clusters of v6.2.0 or later versions, TiDB Cloud Dedicated supports restoring user accounts and SQL bindings from backups by default. +- TiDB Cloud Dedicated does not support restoring system variables stored in the `mysql` schema. +- It is recommended that you import data first, then perform a **manual** snapshot backup, and finally enable Point-in-time Restore. Because the data imported through the TiDB Cloud console **does not** generate change logs, it cannot be automatically detected and backed up. For more information, see [Import CSV Files from Amazon S3 or GCS into TiDB Cloud](/tidb-cloud/import-csv-files.md). +- If you turn on and off Point-in-time Restore multiple times, you can only choose a time point within the recoverable range after the most recent Point-in-time Restore is enabled. The earlier recoverable range is not accessible. +- DO NOT modify the switches of **Point-in-time Restore** and **Dual Region Backup** at the same time. ## Backup -TiDB Dedicated supports automatic backup and manual backup. +### Turn on auto backup -Automatic backups are scheduled for your TiDB Dedicated clusters according to the backup setting, which can reduce your loss in extreme disaster situations. You can also pick a backup snapshot and restore it into a new TiDB Dedicated cluster at any time. +TiDB Cloud Dedicated supports both [snapshot backup](https://docs.pingcap.com/tidb/stable/br-snapshot-guide) and [log backup](https://docs.pingcap.com/tidb/stable/br-pitr-guide). Snapshot backup enables you to restore data to the backup point. By default, snapshot backups are taken automatically and stored according to your backup retention policy. You can disable auto backup at any time. -### Automatic backup +#### Turn on Point-in-time Restore -By the automatic backup, you can back up the TiDB Dedicated cluster data every day at the backup time you have set. To set the backup time, perform the following steps: +> **Note** +> +> The Point-in-time Restore feature is supported for TiDB Cloud Dedicated clusters that are v6.4.0 or later. + +This feature supports restoring data of any point in time to a new cluster. You can use it to: + +- Reduce RPO in disaster recovery. +- Resolve cases of data write errors by restoring point-in-time that is before the error event. +- Audit the historical data of the business. + +It is strongly recommended to turn on this feature. The cost is the same as snapshot backup. For more information, refer to [Data Backup Cost](https://www.pingcap.com/tidb-dedicated-pricing-details#backup-storage-cost). + +To turn on this feature, perform the following steps: + +1. Navigate to the **Backup** page of a TiDB Cloud Dedicated cluster. + +2. Click **Backup Settings**. + +3. Toggle the **Auto Backup** switch to **On**. + +4. Toggle the **Point-in-time Restore** switch to **On**. + + > **Warning** + > + > Point-in-Time Restore only takes effect after the next backup task is completed. To make it take effect earlier, you can [manually perform a backup](#perform-a-manual-backup) after enabling it. + +5. Click **Confirm** to preview the configuration changes. + +6. Click **Confirm** again to save changes. + +#### Configure backup schedule + +TiDB Cloud Dedicated supports daily and weekly backup schedules. By default, the backup schedule is set to daily. You can choose a specific time of the day or week to start snapshot backup. -1. Navigate to the **Backup** page of a TiDB Dedicated cluster. +To configure the backup schedule, perform the following steps: -2. Click **Backup Settings**. The setting window displays. +1. Navigate to the **Backup** page of a TiDB Cloud Dedicated cluster. -3. In the setting window, configure the automatic backup: +2. Click **Backup Settings**. - - Toggle the **Auto Backup** switch to **On**. +3. Toggle the **Auto Backup** switch to **On**. - - In **Backup Cycle**, select either the **Daily** or **Weekly** checkbox. If you select **Weekly**, you need to specify the days of the week for the backup. +4. Configure the backup schedule as follows: + + - In **Backup Scheduler**, select either the **Daily** or **Weekly** checkbox. If you select **Weekly**, you need to specify the days of the week for the backup. + + > **Warning** + > + > - When weekly backup is enabled, the Point-in-time Restore feature is enabled by default and cannot be disabled. + > - If you change the backup scheduler from weekly to daily, the Point-in-time Restore feature remains its original setting. You can manually disable it if needed. - In **Backup Time**, schedule a start time for the daily or weekly cluster backup. - It is recommended to schedule automatic backup at a low workload period. If you do not specify a preferred backup time, TiDB Cloud assigns a default backup time, which is 2:00 AM in the time zone of the region where the cluster is located. + If you do not specify a preferred backup time, TiDB Cloud assigns a default backup time, which is 2:00 AM in the time zone of the region where the cluster is located. - - In **Backup Retention**, configure the minimum backup data retention period. + > **Note** + > + > - Backup jobs are automatically delayed when data import jobs are in progress. **DO NOT** run manual backups during data import or cluster scaling. - - Turn on or off the PITR (**Point-in-time Recovery**) feature. + - In **Backup Retention**, configure the minimum backup data retention period. The default period is 7 days. To minimize the impact on business, it is recommended to schedule automatic backup during periods of low workloads. - > **Note:** + > **Note** > - > - The PITR feature is enabled by default and cannot be disabled when you enable weekly backup. - > - If you change the backup cycle from weekly to daily, the PITR feature remains enabled. You can manually disable it if needed. + > - All auto-backups, except the latest one, will be deleted if their lifetime exceeds the retention period. The latest auto-backup will not be deleted unless you delete it manually. This ensures that you can restore cluster data if accidental deletion occurs. + > - After you delete a cluster, auto-backups with a lifetime within the retention period will be moved to the recycle bin. + > - After you delete a cluster, existing manual backups will be retained until manually deleted or your account is closed. - PITR supports restoring data of any point in time to a new cluster. You can use it to: +### Turn on dual region backup - - Reduce RPO in disaster recovery. - - Resolve cases of data write errors by restoring point-in-time that is before the error event. - - Audit the historical data of the business. +> **Note:** +> +> TiDB Cloud Dedicated clusters hosted on Google Cloud work seamlessly with Google Cloud Storage. Similar to Google Cloud Storage, **TiDB Cloud Dedicated supports dual-region pairing only within the same multi-region code as Google dual-region storage**. For example, in Asia, currently you must pair Tokyo and Osaka together for dual-region storage. For more information, refer to [Dual-regions](https://cloud.google.com/storage/docs/locations#location-dr). - If you have one of the preceding needs and want to use the PITR feature, make sure that your TiDB Dedicated cluster version is at least v6.4.0. +TiDB Cloud Dedicated supports dual region backup by replicating backups from your cluster region to another different region. After you enable this feature, all backups are automatically replicated to the specified region. This provides cross-region data protection and disaster recovery capabilities. It is estimated that approximately 99% of the data can be replicated to the secondary region within an hour. - - In **Backup Storage Region**, select the regions where you want to store your backup data. +Dual region backup costs include both backup storage usage and cross-region data transfer fees. For more information, refer to [Data Backup Cost](https://www.pingcap.com/tidb-dedicated-pricing-details#backup-storage-cost). - TiDB Cloud stores your backup data in the current region of your cluster by default. This behavior cannot be changed. In addition, you can add another remote region, and TiDB Cloud will copy all new backup data to the remote region, which facilitates data safety and faster recovery. After adding a remote region as a backup data storage, you cannot remove the region. +To turn on dual region backup, perform the following steps: -4. Click **Confirm** to preview the configuration change. +1. Navigate to the **Backup** page of a TiDB Cloud Dedicated cluster. - If you turn on PITR, you can select the **Perform a backup immediately and use it as recovery starting point in PITR.** checkbox. Otherwise, PITR will not be available until the next backup is completed. +2. Click **Backup Settings**. -5. Click **Confirm**. +3. Toggle the **Dual Region Backup** switch to **On**. -### Backup storage region support +4. From the **Dual Region** drop-down list, select a region to store the backup files. -Currently, you cannot select an arbitrary remote region for backup data storage. The regions already supported are as follows: +5. Click **Confirm** to preview the configuration changes. -| Cloud provider | Cluster region | Remote region support | -|----------------|-----------------------------|--------------------------| -| Google Cloud | Tokyo (asia-northeast1) | Osaka (asia-northeast2) | +6. Click **Confirm** again to save changes. -> **Note:** +### Turn off auto backup + +> **Note** +> +> Turning off auto backup will also turn off point-in-time restore by default. + +To turn off auto backup, perform the following steps: + +1. Navigate to the **Backup** page of a TiDB Cloud Dedicated cluster. + +2. Click **Backup Settings**. + +3. Toggle the **Auto Backup** switch to **Off**. + +4. Click **Confirm** to preview the configuration changes. + +5. Click **Confirm** again to save changes. + +### Turn off dual region backup + +> **Tip** > -> If you select multiple backup storage regions, you will be charged for multiple backup storage and inter-region backup data replication out from the cluster region to each destination region. The cost is on a per-region basis and varies with the backup regions selected. For more information, see [Data Backup Cost](https://www.pingcap.com/tidb-cloud-pricing-details/#backup-storage-cost). +> Disabling dual region backup does not immediately delete the backups in the secondary region. These backups will be cleaned up later according to the backup retention schedule. To remove them immediately, you can manually [delete the backups](#delete-backups). + +To turn off dual region backup, perform the following steps: + +1. Navigate to the **Backup** page of a TiDB Cloud Dedicated cluster. + +2. Click **Backup Settings**. -### Manual backup +3. Toggle the **Dual Region Backup** switch to **Off**. + +4. Click **Confirm** to preview the configuration changes. + +5. Click **Confirm** again to save changes. + +### Perform a manual backup Manual backups are user-initiated backups that enable you to back up your data to a known state as needed, and then restore to that state at any time. -To apply a manual backup to your TiDB Dedicated cluster, perform the following steps: +To apply a manual backup to your TiDB Cloud Dedicated cluster, perform the following steps: 1. Navigate to the **Backup** tab of a cluster. @@ -94,7 +169,9 @@ To apply a manual backup to your TiDB Dedicated cluster, perform the following s 4. Click **Confirm**. Then your cluster data is backed up. -### Delete backup files +### Delete backups + +#### Delete backup files To delete an existing backup file, perform the following steps: @@ -102,7 +179,7 @@ To delete an existing backup file, perform the following steps: 2. Click **Delete** for the backup file that you want to delete. -### Delete a running backup job +#### Delete a running backup job To delete a running backup job, it is similar as [**Delete backup files**](#delete-backup-files). @@ -110,36 +187,35 @@ To delete a running backup job, it is similar as [**Delete backup files**](#dele 2. Click **Delete** for the backup file that is in the **Pending** or **Running** state. -### Best practices for backup - -- It is recommended that you perform backup operations at cluster idle time to minimize the impact on business. -- Do not run the manual backup while importing data, or during cluster scaling. -- After you delete a cluster, the existing manual backup files will be retained until you manually delete them, or your account is closed. Automatic backup files will be retained for 31 days from the date of cluster deletion. You need to delete the backup files accordingly. - ## Restore -TiDB Dedicated provides two types of data restoration: - -- Restore backup data to a new cluster -- Restore a deleted cluster from the recycle bin - ### Restore data to a new cluster -To restore your TiDB Dedicated cluster data from a backup to a new cluster, take the following steps: +> **Note** +> +> When you restore a TiDB cluster from backups, the restore process retains the original time zone setting without overwriting it. + +To restore your TiDB Cloud Dedicated cluster data from a backup to a new cluster, take the following steps: 1. Navigate to the **Backup** tab of a cluster. 2. Click **Restore**. The setting window displays. -3. In **Restore Mode**, you can choose to restore data of any point in time or a selected backup to a new cluster. +3. In **Restore Mode**, choose **Restore From Region**, indicating the region of backup stores. + + > **Note** + > + > - The default value of the **Restore From Region** is the same as the backup cluster. + +4. In **Restore Mode**, choose to restore data of any point in time or a selected backup to a new cluster.
    - To restore data of any point in time within the backup retention to a new cluster, make sure that **PITR** in **Backup Settings** is on and then take the following steps: + To restore data of any point in time within the backup retention to a new cluster, make sure that **Point-in-time Restore** in **Backup Settings** is on and then take the following steps: - 1. Click **Select Time Point**. - 2. Select **Date** and **Time** you want to restore to. + - Click **Select Time Point**. + - Select **Date** and **Time** you want to restore to.
    @@ -147,28 +223,32 @@ To restore your TiDB Dedicated cluster data from a backup to a new cluster, take To restore a selected backup to the new cluster, take the following steps: - 1. Click **Select Backup Name**. - 2. Select a backup you want to restore to. + - Click **Select Backup Name**. + - Select a backup you want to restore to.
    -4. In **Restore to Region**, select the same region as the **Backup Storage Region** configured in the **Backup Settings**. +5. In **Restore to Region**, select the same region as the **Backup Storage Region** configured in the **Backup Settings**. -5. In the **Restore** window, you can also make the following changes if necessary: +6. In the **Restore** window, you can also make the following changes if necessary: - Set the cluster name. - Update the port number of the cluster. - Increase node number, vCPU and RAM, and storage for the cluster. -6. Click **Restore**. +7. Click **Restore**. - The cluster restore process starts and the **Security Settings** dialog box is displayed. + The cluster restore process starts and the **Password Settings** dialog box is displayed. -7. In the **Security Settings** dialog box, set the root password and allowed IP addresses to connect to your cluster, and then click **Apply**. +8. In the **Password Settings** dialog box, set the root password to connect to your cluster, and then click **Save**. ### Restore a deleted cluster +> **Note:** +> +> You cannot restore a deleted cluster to any point in time. You can only select an automatic or manual backup to restore. + To restore a deleted cluster from recycle bin, take the following steps: 1. Log in to the [TiDB Cloud console](https://tidbcloud.com). @@ -182,10 +262,6 @@ To restore a deleted cluster from recycle bin, take the following steps: 6. Click **Confirm**. - The cluster restore process starts and the **Security Settings** dialog box is displayed. - -7. In the **Security Settings** dialog box, set the root password and allowed IP addresses to connect to your cluster, and then click **Apply**. + The cluster restore process starts and the **Password Settings** dialog box is displayed. -> **Note:** -> -> You cannot restore a deleted cluster to any point in time. You can only select an automatic or manual backup to restore. +7. In the **Password Settings** dialog box, set the root password to connect to your cluster, and then click **Save**. diff --git a/tidb-cloud/branch-github-integration.md b/tidb-cloud/branch-github-integration.md index c6114d0024936..ba2a4f8b23c29 100644 --- a/tidb-cloud/branch-github-integration.md +++ b/tidb-cloud/branch-github-integration.md @@ -1,21 +1,21 @@ --- -title: Integrate TiDB Serverless Branching (Beta) with GitHub -summary: Learn how to integrate the TiDB Serverless branching feature with GitHub. +title: Integrate TiDB Cloud Serverless Branching (Beta) with GitHub +summary: Learn how to integrate the TiDB Cloud Serverless branching feature with GitHub. --- -# Integrate TiDB Serverless Branching (Beta) with GitHub +# Integrate TiDB Cloud Serverless Branching (Beta) with GitHub > **Note:** > -> The integration is built upon [TiDB Serverless branching](/tidb-cloud/branch-overview.md). Make sure that you are familiar with TiDB Serverless branching before reading this document. +> The integration is built upon [TiDB Cloud Serverless branching](/tidb-cloud/branch-overview.md). Make sure that you are familiar with TiDB Cloud Serverless branching before reading this document. -If you use GitHub for application development, you can integrate TiDB Serverless branching into your GitHub CI/CD pipeline, which lets you automatically test your pull requests with branches without affecting the production database. +If you use GitHub for application development, you can integrate TiDB Cloud Serverless branching into your GitHub CI/CD pipeline, which lets you automatically test your pull requests with branches without affecting the production database. -In the integration process, you will be prompted to install the [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) GitHub App. The app can automatically manage TiDB Serverless branches according to pull requests in your GitHub repository. For example, when you create a pull request, the app will create a corresponding branch for your TiDB Serverless cluster, in which you can work on new features or bug fixes in isolation without affecting the production database. +In the integration process, you will be prompted to install the [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) GitHub App. The app can automatically manage TiDB Cloud Serverless branches according to pull requests in your GitHub repository. For example, when you create a pull request, the app will create a corresponding branch for your TiDB Cloud Serverless cluster, in which you can work on new features or bug fixes in isolation without affecting the production database. This document covers the following topics: -1. How to integrate TiDB Serverless branching with GitHub +1. How to integrate TiDB Cloud Serverless branching with GitHub 2. How does the TiDB Cloud Branching app work 3. How to build a branching-based CI workflow to test every pull request using branches rather than the production cluster @@ -25,13 +25,13 @@ Before the integration, make sure that you have the following: - A GitHub account - A GitHub repository for your application -- A [TiDB Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) +- A [TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) -## Integrate TiDB Serverless branching with your GitHub repository +## Integrate TiDB Cloud Serverless branching with your GitHub repository -To integrate TiDB Serverless branching with your GitHub repository, take the following steps: +To integrate TiDB Cloud Serverless branching with your GitHub repository, take the following steps: -1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Serverless cluster to go to its overview page. +1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Cloud Serverless cluster to go to its overview page. 2. Click **Branches** in the left navigation pane. @@ -40,7 +40,7 @@ To integrate TiDB Serverless branching with your GitHub repository, take the fol - If you have not logged into GitHub, you will be asked to log into GitHub first. - If it is the first time you use the integration, you will be asked to authorize the **TiDB Cloud Branching** app. - + 4. In the **Connect to GitHub** dialog, select a GitHub account in the **GitHub Account** drop-down list. @@ -48,18 +48,18 @@ To integrate TiDB Serverless branching with your GitHub repository, take the fol 5. Select your target repository in the **GitHub Repository** drop-down list. If the list is long, you can search the repository by typing the name. -6. Click **Connect** to connect between your TiDB Serverless cluster and your GitHub repository. +6. Click **Connect** to connect between your TiDB Cloud Serverless cluster and your GitHub repository. - + ## TiDB Cloud Branching app behaviors -After you connect your TiDB Serverless cluster to your GitHub repository, for each pull request in this repository, the [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) GitHub App can automatically manage its corresponding TiDB Serverless branch. The following lists the default behaviors for pull request changes: +After you connect your TiDB Cloud Serverless cluster to your GitHub repository, for each pull request in this repository, the [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) GitHub App can automatically manage its corresponding TiDB Cloud Serverless branch. The following lists the default behaviors for pull request changes: | Pull request changes | TiDB Cloud Branching app behaviors | |------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Create a pull request | When you create a pull request in the repository, the [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) app creates a branch for your TiDB Serverless cluster. The branch name is in the `${github_branch_name}_${pr_id}_${commit_sha}` format. Note that the number of branches has a [limit](/tidb-cloud/branch-overview.md#limitations-and-quotas). | -| Push new commits to a pull request | Every time you push a new commit to a pull request in the repository, the [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) app deletes the previous TiDB Serverless branch and creates a new branch for the latest commit. | +| Create a pull request | When you create a pull request in the repository, the [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) app creates a branch for your TiDB Cloud Serverless cluster. When `branch.mode` is set to `reset`, the branch name follows the `${github_branch_name}_${pr_id}` format. When `branch.mode` is set to `reserve`, the branch name follows the `${github_branch_name}_${pr_id}_${commit_sha}` format. Note that the number of branches has a [limit](/tidb-cloud/branch-overview.md#limitations-and-quotas). | +| Push new commits to a pull request | When `branch.mode` is set to `reset`, every time you push a new commit to a pull request in the repository, the [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) app resets the TiDB Cloud Serverless branch. When `branch.mode` is set to `reserve`, the app creates a new branch for the latest commit. | | Close or merge a pull request | When you close or merge a pull request, the [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) app deletes the branch for this pull request. | | Reopen a pull request | When you reopen a pull request, the [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) app creates a branch for the lasted commit of the pull request. | @@ -94,23 +94,26 @@ github: - ".*_db" ``` -### branch.autoReserved +### branch.mode -**Type:** boolean. **Default:** `false`. +**Type:** string. **Default:** `reset`. -If it is set to `true`, the TiDB Cloud Branching app will not delete the TiDB Serverless branch that is created in the previous commit. +Specify how the TiDB Cloud Branching app handles branch updates: + +- If it is set to `reset`, the TiDB Cloud Branching app will update the existing branch with the latest data. +- If it is set to `reserve`, the TiDB Cloud Branching app will create a new branch for your latest commit. ```yaml github: branch: - autoReserved: false + mode: reset ``` ### branch.autoDestroy **Type:** boolean. **Default:** `true`. -If it is set to `false`, the TiDB Cloud Branching app will not delete the TiDB Serverless branch when a pull request is closed or merged. +If it is set to `false`, the TiDB Cloud Branching app will not delete the TiDB Cloud Serverless branch when a pull request is closed or merged. ```yaml github: @@ -120,21 +123,21 @@ github: ## Create a branching CI workflow -One of the best practices for using branches is to create a branching CI workflow. With the workflow, you can test your code using a TiDB Serverless branch instead of using the production cluster before merging the pull request. You can find a live demo [here](https://github.com/shiyuhang0/tidbcloud-branch-gorm-example). +One of the best practices for using branches is to create a branching CI workflow. With the workflow, you can test your code using a TiDB Cloud Serverless branch instead of using the production cluster before merging the pull request. You can find a live demo [here](https://github.com/shiyuhang0/tidbcloud-branch-gorm-example). Here are the main steps to create the workflow: -1. [Integrate TiDB Serverless branching with your GitHub repository](#integrate-tidb-serverless-branching-with-your-github-repository). +1. [Integrate TiDB Cloud Serverless branching with your GitHub repository](#integrate-tidb-cloud-serverless-branching-with-your-github-repository). 2. Get the branch connection information. - You can use the [wait-for-tidbcloud-branch](https://github.com/tidbcloud/wait-for-tidbcloud-branch) action to wait for the readiness of the TiDB Serverless branch and get the connection information of the branch. + You can use the [wait-for-tidbcloud-branch](https://github.com/tidbcloud/wait-for-tidbcloud-branch) action to wait for the readiness of the TiDB Cloud Serverless branch and get the connection information of the branch. Example usage: ```yaml steps: - - name: Wait for TiDB Serverless branch to be ready + - name: Wait for TiDB Cloud Serverless branch to be ready uses: tidbcloud/wait-for-tidbcloud-branch@v0 id: wait-for-branch with: @@ -142,12 +145,15 @@ Here are the main steps to create the workflow: public-key: ${{ secrets.TIDB_CLOUD_API_PUBLIC_KEY }} private-key: ${{ secrets.TIDB_CLOUD_API_PRIVATE_KEY }} - - name: Test with TiDB Serverless branch + - name: Test with TiDB Cloud Serverless branch run: | echo "The host is ${{ steps.wait-for-branch.outputs.host }}" echo "The user is ${{ steps.wait-for-branch.outputs.user }}" echo "The password is ${{ steps.wait-for-branch.outputs.password }}" ``` + + - `token`: GitHub will automatically create a [GITHUB_TOKEN](https://docs.github.com/en/actions/security-guides/automatic-token-authentication) secret. You can use it directly. + - `public-key` and `private-key`: The TiDB Cloud [API key](https://docs.pingcap.com/tidbcloud/api/v1beta#section/Authentication/API-Key-Management). 3. Modify your test code. @@ -155,4 +161,10 @@ Here are the main steps to create the workflow: ## What's next +Learn how to use the branching GitHub integration with the following examples: + +- [branching-gorm-example](https://github.com/tidbcloud/branching-gorm-example) +- [branching-django-example](https://github.com/tidbcloud/branching-django-example) +- [branching-rails-example](https://github.com/tidbcloud/branching-rails-example) + You can also build your branching CI/CD workflow without the branching GitHub integration. For example, you can use [`setup-tidbcloud-cli`](https://github.com/tidbcloud/setup-tidbcloud-cli) and GitHub Actions to customize your CI/CD workflows. diff --git a/tidb-cloud/branch-manage.md b/tidb-cloud/branch-manage.md index 86978c2391468..da9f3b88a36a8 100644 --- a/tidb-cloud/branch-manage.md +++ b/tidb-cloud/branch-manage.md @@ -1,11 +1,11 @@ --- -title: Manage TiDB Serverless Branches -summary: Learn How to manage TiDB Serverless branches. +title: Manage TiDB Cloud Serverless Branches +summary: Learn How to manage TiDB Cloud Serverless branches. --- -# Manage TiDB Serverless Branches +# Manage TiDB Cloud Serverless Branches -This document describes how to manage TiDB Serverless branches using the [TiDB Cloud console](https://tidbcloud.com). To manage it using the TiDB Cloud CLI, see [`ticloud branch`](/tidb-cloud/ticloud-branch-create.md). +This document describes how to manage TiDB Cloud Serverless branches using the [TiDB Cloud console](https://tidbcloud.com). To manage it using the TiDB Cloud CLI, see [`ticloud branch`](/tidb-cloud/ticloud-branch-create.md). ## Required access @@ -18,14 +18,25 @@ For more information about permissions, see [User roles](/tidb-cloud/manage-user > **Note:** > -> You can only create branches for TiDB Serverless clusters that are created after July 5, 2023. See [Limitations and quotas](/tidb-cloud/branch-overview.md#limitations-and-quotas) for more limitations. +> You can only create branches for TiDB Cloud Serverless clusters that are created after July 5, 2023. See [Limitations and quotas](/tidb-cloud/branch-overview.md#limitations-and-quotas) for more limitations. To create a branch, perform the following steps: -1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Serverless cluster to go to its overview page. +1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Cloud Serverless cluster to go to its overview page. 2. Click **Branches** in the left navigation pane. -3. Click **Create Branch** in the upper-right corner. -4. Enter the branch name, and then click **Create**. +3. In the upper-right corner of the **Branches** page, click **Create Branch**. A dialog is displayed. + + Alternatively, to create a branch from an existing parent branch, locate the row of your target parent branch, and then click **...** > **Create Branch** in the **Action** column. + +4. In the **Create Branch** dialog, configure the following options: + + - **Name**: enter a name for the branch. + - **Parent branch**: select the original cluster or an existing branch. `main` represents the current cluster. + - **Include data up to**: choose one of the following: + - **Current point in time**: create a branch from the current state. + - **Specific date and time**: create a branch from a specified time. + +5. Click **Create**. Depending on the data size in your cluster, the branch creation will be completed in a few minutes. @@ -33,7 +44,7 @@ Depending on the data size in your cluster, the branch creation will be complete To view branches for your cluster, perform the following steps: -1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Serverless cluster to go to its overview page. +1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Cloud Serverless cluster to go to its overview page. 2. Click **Branches** in the left navigation pane. The branch list of the cluster is displayed in the right pane. @@ -42,27 +53,47 @@ To view branches for your cluster, perform the following steps: To connect to a branch, perform the following steps: -1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Serverless cluster to go to its overview page. +1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Cloud Serverless cluster to go to its overview page. 2. Click **Branches** in the left navigation pane. 3. In the row of your target branch to be connected, click **...** in the **Action** column. 4. Click **Connect** in the drop-down list. The dialog for the connection information is displayed. -5. Click **Create password** or **Reset password** to create or reset the root password. +5. Click **Generate Password** or **Reset Password** to create or reset the root password. 6. Connect to the branch using the connection information. -> **Note:** -> -> Currently, branches do not support [private endpoints](/tidb-cloud/set-up-private-endpoint-connections-serverless.md). +Alternatively, you can get the connection string from the cluster overview page: + +1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Cloud Serverless cluster to go to its overview page. +2. Click **Connect** in the upper-right corner. +3. Select the branch you want to connect to in the `Branch` drop-down list. +4. Click **Generate Password** or **Reset Password** to create or reset the root password. +5. Connect to the branch using the connection information. ## Delete a branch To delete a branch, perform the following steps: -1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Serverless cluster to go to its overview page. +1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Cloud Serverless cluster to go to its overview page. 2. Click **Branches** in the left navigation pane. 3. In the row of your target branch to be deleted, click **...** in the **Action** column. 4. Click **Delete** in the drop-down list. 5. Confirm the deletion. +## Reset a branch + +Resetting a branch synchronizes it with the latest data from its parent. + +> **Note:** +> +> This operation is irreversible. Before resetting a branch, make sure that you have backed up any important data. + +To reset a branch, perform the following steps: + +1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Cloud Serverless cluster to go to its overview page. +2. Click **Branches** in the left navigation pane. +3. In the row of your target branch to be reset, click **...** in the **Action** column. +4. Click **Reset** in the drop-down list. +5. Confirm the reset. + ## What's next -- [Integrate TiDB Serverless branching into your GitHub CI/CD pipeline](/tidb-cloud/branch-github-integration.md) +- [Integrate TiDB Cloud Serverless branching into your GitHub CI/CD pipeline](/tidb-cloud/branch-github-integration.md) diff --git a/tidb-cloud/branch-overview.md b/tidb-cloud/branch-overview.md index aa75153ea68ff..6908ebf402984 100644 --- a/tidb-cloud/branch-overview.md +++ b/tidb-cloud/branch-overview.md @@ -1,19 +1,19 @@ --- -title: TiDB Serverless Branching (Beta) Overview -summary: Learn the concept of TiDB Serverless branches. +title: TiDB Cloud Serverless Branching (Beta) Overview +summary: Learn the concept of TiDB Cloud Serverless branches. --- -# TiDB Serverless Branching (Beta) Overview +# TiDB Cloud Serverless Branching (Beta) Overview -TiDB Cloud lets you create branches for TiDB Serverless clusters. A branch for a cluster is a separate instance that contains a diverged copy of data from the original cluster. It provides an isolated environment, allowing you to experiment freely without worrying about affecting the original cluster. +TiDB Cloud lets you create branches for TiDB Cloud Serverless clusters. A branch for a cluster is a separate instance that contains a diverged copy of data from the original cluster. It provides an isolated environment, allowing you to experiment freely without worrying about affecting the original cluster. -With TiDB Serverless branches, developers can work in parallel, iterate rapidly on new features, troubleshoot issues without affecting the production database, and easily revert changes if needed. This feature streamlines the development and deployment process while ensuring a high level of stability and reliability for the production database. +With TiDB Cloud Serverless branches, developers can work in parallel, iterate rapidly on new features, troubleshoot issues without affecting the production database, and easily revert changes if needed. This feature streamlines the development and deployment process while ensuring a high level of stability and reliability for the production database. ## Implementations -When a branch for a cluster is created, the data in the branch diverges from the original cluster. This means that subsequent changes made in either the original cluster or the branch will not be synchronized with each other. +When a branch for a cluster is created, the data in the branch diverges from the original cluster or its parent branch at a specific point in time. This means that subsequent changes made in either the parent or the branch will not be synchronized with each other. -To ensure fast and seamless branch creation, TiDB Serverless uses a copy-on-write technique for sharing data between the original cluster and its branches. This process usually completes within a few minutes and is imperceptible to users, ensuring that it does not affect the performance of your original cluster. +To ensure fast and seamless branch creation, TiDB Cloud Serverless uses a copy-on-write technique for sharing data between the original cluster and its branches. This process usually completes within a few minutes and is imperceptible to users, ensuring that it does not affect the performance of your original cluster. ## Scenarios @@ -33,13 +33,18 @@ You can create branches easily and quickly to get isolated data environments. Br ## Limitations and quotas -Currently, TiDB Serverless branches are in beta and free of charge. +Currently, TiDB Cloud Serverless branches are in beta and free of charge. -- You can only create branches for TiDB Serverless clusters that are created after July 5, 2023. +- You can only create branches for TiDB Cloud Serverless clusters that are created after July 5, 2023. -- For each organization in TiDB Cloud, you can create a maximum of five TiDB Serverless branches by default across all the clusters. The branches of a cluster will be created in the same region as the cluster, and you cannot create branches for a throttled cluster or a cluster larger than 100 GiB. +- For each organization in TiDB Cloud, you can create a maximum of five TiDB Cloud Serverless branches by default across all the clusters. The branches of a cluster will be created in the same region as the cluster, and you cannot create branches for a throttled cluster or a cluster larger than 100 GiB. -- For each branch, 5 GiB storage is allowed. Once the storage is reached, the read and write operations on this branch will be throttled until you reduce the storage. +- For each branch of a free cluster, 10 GiB storage is allowed. For each branch of a scalable cluster, 100 GiB storage is allowed. Once the storage is reached, the read and write operations on this branch will be throttled until you reduce the storage. + +- When [creating a branch](/tidb-cloud/branch-manage.md#create-a-branch) from a specific point in time: + + - For branches of a free cluster, you can select any time within the last 24 hours. + - For branches of a scalable cluster, you can select any time within the last 14 days. If you need more quotas, [contact TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). diff --git a/tidb-cloud/built-in-monitoring.md b/tidb-cloud/built-in-monitoring.md index fc18d7d7f1903..ff039fdc176cc 100644 --- a/tidb-cloud/built-in-monitoring.md +++ b/tidb-cloud/built-in-monitoring.md @@ -22,11 +22,11 @@ To view the metrics on the Metrics page, take the following steps: ## Metrics retention policy -For TiDB Dedicated clusters and TiDB Serverless clusters, the metrics data is kept for 7 days. +For TiDB Cloud Dedicated clusters and TiDB Cloud Serverless clusters, the metrics data is kept for 7 days. -## Metrics for TiDB Dedicated clusters +## Metrics for TiDB Cloud Dedicated clusters -The following sections illustrate the metrics on the Metrics page for TiDB Dedicated clusters. +The following sections illustrate the metrics on the Metrics page for TiDB Cloud Dedicated clusters. ### Overview @@ -67,22 +67,22 @@ The following sections illustrate the metrics on the Metrics page for TiDB Dedic | Metric name | Labels | Description | | :------------| :------| :-------------------------------------------- | | TiDB Uptime | node | The runtime of each TiDB node since last restart. | -| TiDB CPU Usage | node | The CPU usage statistics of each TiDB node. | -| TiDB Memory Usage | node | The memory usage statistics of each TiDB node. | +| TiDB CPU Usage | node, limit | The CPU usage statistics or upper limit of each TiDB node. | +| TiDB Memory Usage | node, limit | The memory usage statistics or upper limit of each TiDB node. | | TiKV Uptime | node | The runtime of each TiKV node since last restart. | -| TiKV CPU Usage | node | The CPU usage statistics of each TiKV node. | -| TiKV Memory Usage | node | The memory usage statistics of each TiKV node. | +| TiKV CPU Usage | node, limit | The CPU usage statistics or upper limit of each TiKV node. | +| TiKV Memory Usage | node, limit | The memory usage statistics or upper limit of each TiKV node. | | TiKV IO Bps | node-write, node-read | The total input/output bytes per second of read and write in each TiKV node. | -| TiKV Storage Usage | node | The storage usage statistics of each TiKV node. | +| TiKV Storage Usage | node, limit | The storage usage statistics or upper limit of each TiKV node. | | TiFlash Uptime | node | The runtime of each TiFlash node since last restart. | -| TiFlash CPU Usage | node | The CPU usage statistics of each TiFlash node. | -| TiFlash Memory | node | The memory usage statistics of each TiFlash node. | +| TiFlash CPU Usage | node, limit | The CPU usage statistics or upper limit of each TiFlash node. | +| TiFlash Memory Usage | node, limit | The memory usage statistics or upper limit of each TiFlash node. | | TiFlash IO MBps | node-write, node-read | The total bytes of read and write in each TiFlash node. | -| TiFlash Storage Usage | node | The storage usage statistics of each TiFlash node. | +| TiFlash Storage Usage | node, limit | The storage usage statistics or upper limit of each TiFlash node. | -## Metrics for TiDB Serverless clusters +## Metrics for TiDB Cloud Serverless clusters -The Metrics page provides two tabs for metrics of TiDB Serverless clusters: +The Metrics page provides two tabs for metrics of TiDB Cloud Serverless clusters: - Cluster Status: displays the cluster-level main metrics. - Database Status: displays the database-level main metrics. @@ -96,11 +96,11 @@ The following table illustrates the cluster-level main metrics under the **Clust | Request Units | RU per second | The Request Unit (RU) is a unit of measurement used to track the resource consumption of a query or transaction. In addition to queries that you run, Request Units can be consumed by background activities, so when the QPS is 0, the Request Units per second might not be zero. | | Used Storage Size | Row-based storage, Columnar storage | The size of the row store and the size of the column store. | | Query Per Second | All, {SQL type} | The number of SQL statements executed per second, which are collected by SQL types, such as `SELECT`, `INSERT`, and `UPDATE`. | -| Average Query Duration | All, {SQL type} | The duration from receiving a request from the client to the TiDB Serverless cluster until the cluster executes the request and returns the result to the client. | +| Average Query Duration | All, {SQL type} | The duration from receiving a request from the client to the TiDB Cloud Serverless cluster until the cluster executes the request and returns the result to the client. | | Failed Query | All | The number of SQL statement execution errors per second. | | Transaction Per Second | All | The number of transactions executed per second. | | Average Transaction Duration | All | The average execution duration of transactions. | -| Total Connection | All | The number of connections to the TiDB Serverless cluster. | +| Total Connection | All | The number of connections to the TiDB Cloud Serverless cluster. | ### Database Status diff --git a/tidb-cloud/changefeed-overview.md b/tidb-cloud/changefeed-overview.md index b81f74f4166dd..0570e5a9659cf 100644 --- a/tidb-cloud/changefeed-overview.md +++ b/tidb-cloud/changefeed-overview.md @@ -9,9 +9,9 @@ TiDB Cloud changefeed helps you stream data from TiDB Cloud to other data servic > **Note:** > -> - To use the changefeed feature, make sure that your TiDB Dedicated cluster version is v6.4.0 or later. -> - Currently, TiDB Cloud only allows up to 5 changefeeds per cluster. -> - For [TiDB Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-serverless), the changefeed feature is unavailable. +> - Currently, TiDB Cloud only allows up to 100 changefeeds per cluster. +> - Currently, TiDB Cloud only allows up to 100 table filter rules per changefeed. +> - For [TiDB Cloud Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless), the changefeed feature is unavailable. To access the changefeed feature, navigate to the cluster overview page of your TiDB cluster, and then click **Changefeed** in the left navigation pane. The changefeed page is displayed. @@ -21,7 +21,7 @@ On the changefeed page, you can create a changefeed, view a list of existing cha To create a changefeed, refer to the tutorials: -- [Sink to Apache Kafka](/tidb-cloud/changefeed-sink-to-apache-kafka.md) (Beta) +- [Sink to Apache Kafka](/tidb-cloud/changefeed-sink-to-apache-kafka.md) - [Sink to MySQL](/tidb-cloud/changefeed-sink-to-mysql.md) - [Sink to TiDB Cloud](/tidb-cloud/changefeed-sink-to-tidb-cloud.md) - [Sink to cloud storage](/tidb-cloud/changefeed-sink-to-cloud-storage.md) @@ -65,8 +65,10 @@ It takes about 10 minutes to complete the scaling process (during which the chan TiDB Cloud populates the changefeed configuration by default. You can modify the following configurations: - - MySQL sink: **MySQL Connection** and **Table Filter**. - - Kafka sink: all configurations. + - Apache Kafka sink: all configurations. + - MySQL sink: **MySQL Connection**, **Table Filter**, and **Event Filter**. + - TiDB Cloud sink: **TiDB Cloud Connection**, **Table Filter**, and **Event Filter**. + - Cloud storage sink: **Storage Endpoint**, **Table Filter**, and **Event Filter**. 4. After editing the configuration, click **...** > **Resume** to resume the corresponding changefeed. @@ -94,4 +96,4 @@ The states are described as follows: - `DELETING`: the replication task is being deleted. - `DELETED`: the replication task is deleted. - `WARNING`: the replication task returns a warning. The replication cannot continue due to some recoverable errors. The changefeed in this state keeps trying to resume until the state transfers to `RUNNING`. The changefeed in this state blocks [GC operations](https://docs.pingcap.com/tidb/stable/garbage-collection-overview). -- `FAILED`: the replication task fails. Due to some unrecoverable errors, the replication task cannot resume and cannot be recovered. The changefeed in this state does not block GC operations. +- `FAILED`: the replication task fails. Due to some errors, the replication task cannot resume and cannot be recovered automatically. If the issues are resolved before the garbage collection (GC) of the incremental data, you can manually resume the failed changefeed. The default Time-To-Live (TTL) duration for incremental data is 24 hours, which means that the GC mechanism does not delete any data within 24 hours after the changefeed is interrupted. diff --git a/tidb-cloud/changefeed-replication.md b/tidb-cloud/changefeed-replication.md deleted file mode 100644 index 4ac211c7f67be..0000000000000 --- a/tidb-cloud/changefeed-replication.md +++ /dev/null @@ -1,117 +0,0 @@ ---- -title: TiDB Cloud Replication -summary: Learn how to create a replica to stream data from a primary TiDB cluster to a secondary TiDB cluster. ---- - -# TiDB Cloud Replication - -TiDB Cloud Replication is a feature that allows you to create a continuously replicated readable secondary TiDB cluster for a primary TiDB cluster in TiDB Cloud. This readable secondary cluster (also known as cross-region replication, secondary replication, or geo-replication) can be in the same region as the primary TiDB cluster, or more commonly, in a different region. - -With TiDB Cloud Replication, you can perform quick disaster recovery of a database in the event of a regional disaster or large-scale failure, which helps achieve business continuity. Once a secondary cluster is set up, you can manually initiate geographic failover to the secondary cluster in a different region. - -> **Warning:** -> -> Currently, the **TiDB Cloud Replication** feature is in beta with the following limitations: -> -> * One primary cluster can only have one replication. -> * You cannot use a secondary cluster as a source of **TiDB Cloud Replication** to another cluster. -> * **TiDB Cloud Replication** contradicts [**Sink to Apache Kafka**](/tidb-cloud/changefeed-sink-to-apache-kafka.md) and [**Sink to MySQL**](/tidb-cloud/changefeed-sink-to-mysql.md). When **TiDB Cloud Replication** is enabled, neither the primary nor the secondary cluster can use the **Sink to Apache Kafka** or **Sink to MySQL** changefeed and vice versa. -> * Because TiDB Cloud uses TiCDC to establish replication, it has the same [restrictions as TiCDC](https://docs.pingcap.com/tidb/stable/ticdc-overview#restrictions). -> * **TiDB Cloud Replication** is only available upon request. To request this feature, click **?** in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com) and click **Request Support**. Then, fill in "Apply for TiDB Cloud Replication" in the **Description** field and click **Send**. - -To support application replication, you must deploy your applications in both primary and secondary regions, and ensure that each application is connected to the TiDB cluster in the same region. The applications in the secondary region are on standby. When the primary region fails, you can initiate a "Detach" operation to make the TiDB cluster in the secondary region active, and then transfer all data traffic to the applications in the secondary region. - -The following diagram illustrates a typical deployment of a geo-redundant cloud application using TiDB Cloud Replication: - - -![TiDB Cloud Replication](/media/tidb-cloud/changefeed-replication-deployment.png) - -Creating a secondary TiDB cluster is only a part of the business continuity solution. To recover an application (or service) end-to-end after a catastrophic failure, you also need to ensure that all components and dependent services of the application can be restored. - -- Check whether each component of the application is resilient to the same failures and become available within recovery time objective (RTO) of your application. The typical components of an application include client software (such as browsers with custom JavaScript), web front ends, storage, and DNS. -- Identify all dependent services, check the guarantees and capabilities of these services, and ensure that your application is operational during a failover of these services. - -## Terminology and capabilities of TiDB Cloud Replication - -### Automatic asynchronous replication - -For each primary TiDB cluster, only one secondary cluster can be created. TiDB Cloud makes a full backup of the primary TiDB cluster and then restores the backup to the newly created secondary cluster, which makes sure that the secondary cluster has all the existing data. After the secondary cluster is created, all data changes on the primary cluster will be replicated asynchronously to the secondary cluster. - -### Readable secondary cluster - -The secondary cluster is in the read-only mode. If you have any read-only workload with low real-time data requirements, you can distribute it to the secondary cluster. - -To satisfy read-intensive scenarios in the same region, you can use **TiDB Cloud Replication** to create a readable secondary cluster in the same region as the primary cluster. However, because a secondary cluster in the same region does not provide additional resiliency for large-scale outages or catastrophic failures, do not use it as a failover target for regional disaster recovery purposes. - -### Planned Detach - -**Planned Detach** can be triggered by you manually. It is used for planned maintenance in most cases, such as disaster recovery drills. **Planned detach** makes sure that all data changes are replicated to the secondary cluster without data loss (RPO=0). For RTO, it depends on the replication lag between primary and secondary clusters. In most cases, the RTO is at a level of minutes. - -**Planned Detach** detaches the secondary cluster from the primary cluster into an individual cluster. When **Planned Detach** is triggered, it performs the following steps: - -1. Sets the primary cluster as read-only, to prevent any new transaction from being committed to the primary cluster. -2. Waits until the secondary cluster is fully synced with the primary cluster. -3. Stops the replication from the primary to the secondary cluster. -4. Sets the original secondary cluster as writable, which makes it available to serve your business. - -After **Planned Detach** is finished, the original primary cluster is set as read-only. If you still need to write to the original primary cluster, you can do one of the following to set the cluster as writable explicitly: - -- Go to the cluster details page, click **Settings**, and then click the **Make Writable** drop-down button. -- Connect to the SQL port of the original primary cluster and execute the following statement: - - {{< copyable "sql" >}} - - ```sql - set global tidb_super_read_only=OFF; - ``` - -### Force Detach - -To recover from an unplanned outage, use **Force Detach**. In the event of a catastrophic failure in the region where the primary cluster is located, you should use **Force Detach** so that the secondary cluster can serve the business as quickly as possible, ensuring business continuity. Because this operation makes the secondary cluster serve as an individual cluster immediately and does not wait for any unreplicated data, the RPO depends on the Primary-Secondary replication lag, while the RTO depends on how quickly **Force Detach** is triggered by you. - -**Force Detach** detaches the secondary cluster from the primary cluster into an individual cluster. When **Force Detach** is triggered, it performs the following steps: - -1. Stops data replication from the primary to the secondary cluster immediately. -2. Sets the original secondary cluster as writable so that it can start serving your workload. -3. If the original primary cluster is still accessible, or when the original primary cluster recovers, TiDB Cloud sets it as read-only to avoid any new transaction being committed to it. - -Once the original primary cluster is recovered from the outage, you still have the opportunity to review transactions that have been executed in the original primary cluster but not in the original secondary cluster by comparing the data in the two clusters, and decide whether to manually replicate these unsynchronized transactions to the original secondary cluster based on your business situation. - -The data replication topology between primary and secondary clusters does not exist anymore after you detach the secondary cluster. The original primary cluster is set to the read-only mode and the original secondary cluster becomes writable. If any DML or DDL is planned on the original primary cluster, you need to disable the read-only mode manually on it by doing one of the following: - -- Go to the cluster details page, click **Settings**, and then click the **Make Writable** drop-down button. -- Connect to the SQL port of the original primary cluster and execute the following statement: - - {{< copyable "sql" >}} - - ```sql - set global tidb_super_read_only=OFF; - ``` - -## Configure TiDB Cloud Replication - -To configure TiDB Cloud Replication, do the following: - -1. In the [TiDB Cloud console](https://tidbcloud.com), navigate to the cluster overview page of your TiDB cluster, and then click **Changefeed** in the left navigation pane. -2. Click **Create a replica of your TiDB Cluster**. -3. Fill in the username and password of your database. -4. Choose the region of the secondary cluster. -5. Click **Create**. After a while, the sink will begin its work, and the status of the sink will be changed to "**Producing**". - -To trigger a **Planned Detach** or **Force Detach**, do the following: - -1. In the [TiDB Cloud console](https://tidbcloud.com), navigate to the cluster overview page of your TiDB cluster, and then click **Changefeed** in the left navigation pane. -2. Click **Create a replica of your TiDB Cluster**. -3. Click **Planned Detach** or **Force Detach**. - -## Scale the primary cluster - -You can scale out or scale in the primary cluster without disconnecting the secondary cluster. When the primary cluster is scaled, the secondary cluster follows the same scaling automatically. - -## Monitor the primary-secondary lag - -To monitor lag concerning the RPO, do the following: - -1. In the [TiDB Cloud console](https://tidbcloud.com), navigate to the cluster overview page of your TiDB cluster, and then click **Changefeed** in the left navigation pane. -2. Click **Create a replica of your TiDB Cluster**. -3. You can see the lag of the primary-secondary cluster. diff --git a/tidb-cloud/changefeed-sink-to-apache-kafka.md b/tidb-cloud/changefeed-sink-to-apache-kafka.md index f79b78cc7b376..3d575b47eb7fa 100644 --- a/tidb-cloud/changefeed-sink-to-apache-kafka.md +++ b/tidb-cloud/changefeed-sink-to-apache-kafka.md @@ -1,23 +1,34 @@ --- -title: Sink to Apache Kafka (Beta) -Summary: Learn how to create a changefeed to stream data from TiDB Cloud to Apache Kafka. +title: Sink to Apache Kafka +summary: This document explains how to create a changefeed to stream data from TiDB Cloud to Apache Kafka. It includes restrictions, prerequisites, and steps to configure the changefeed for Apache Kafka. The process involves setting up network connections, adding permissions for Kafka ACL authorization, and configuring the changefeed specification. --- -# Sink to Apache Kafka (Beta) +# Sink to Apache Kafka This document describes how to create a changefeed to stream data from TiDB Cloud to Apache Kafka. > **Note:** > -> - Currently, Kafka sink is in **beta**. To use the changefeed feature, make sure that your TiDB Dedicated cluster version is v6.4.0 or later. -> - For [TiDB Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-serverless), the changefeed feature is unavailable. +> - To use the changefeed feature, make sure that your TiDB Cloud Dedicated cluster version is v6.1.3 or later. +> - For [TiDB Cloud Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless), the changefeed feature is unavailable. ## Restrictions -- For each TiDB Cloud cluster, you can create up to 5 changefeeds. +- For each TiDB Cloud cluster, you can create up to 100 changefeeds. - Currently, TiDB Cloud does not support uploading self-signed TLS certificates to connect to Kafka brokers. - Because TiDB Cloud uses TiCDC to establish changefeeds, it has the same [restrictions as TiCDC](https://docs.pingcap.com/tidb/stable/ticdc-overview#unsupported-scenarios). - If the table to be replicated does not have a primary key or a non-null unique index, the absence of a unique constraint during replication could result in duplicated data being inserted downstream in some retry scenarios. +- If you choose Private Link or Private Service Connect as the network connectivity method, ensure that your TiDB cluster version meets the following requirements: + + - For v6.5.x: version v6.5.9 or later + - For v7.1.x: version v7.1.4 or later + - For v7.5.x: version v7.5.1 or later + - For v8.1.x: all versions of v8.1.x and later are supported +- If you want to use Debezium as your data format, make sure the version of your TiDB cluster is v8.1.0 or later. +- For the partition distribution of Kafka messages, note the following: + + - If you want to distribute changelogs by primary key or index value to Kafka partition with a specified index name, make sure the version of your TiDB cluster is v7.5.0 or later. + - If you want to distribute changelogs by column value to Kafka partition, make sure the version of your TiDB cluster is v7.5.0 or later. ## Prerequisites @@ -28,7 +39,33 @@ Before creating a changefeed to stream data to Apache Kafka, you need to complet ### Network -Make sure that your TiDB cluster can connect to the Apache Kafka service. +Ensure that your TiDB cluster can connect to the Apache Kafka service. You can choose one of the following connection methods: + +- Private Connect (Beta): ideal for avoiding VPC CIDR conflicts and meeting security compliance, but incurs additional [Private Data Link Cost](/tidb-cloud/tidb-cloud-billing-ticdc-rcu.md#private-data-link-cost). +- VPC Peering: suitable as a cost-effective option, but requires managing potential VPC CIDR conflicts and security considerations. +- Public IP: suitable for a quick setup. + + +
    + +Private Connect leverages **Private Link** or **Private Service Connect** technologies from cloud providers to enable resources in your VPC to connect to services in other VPCs using private IP addresses, as if those services were hosted directly within your VPC. + +Currently, TiDB Cloud supports Private Connect for generic Kafka only. It does not include special integration with MSK, Confluent Kafka, or other services. + +- If your Apache Kafka service is hosted in AWS, follow [Set Up Self-Hosted Kafka Private Link Service in AWS](/tidb-cloud/setup-self-hosted-kafka-private-link-service.md) to ensure that the network connection is properly configured. After setup, provide the following information in the TiDB Cloud console to create the changefeed: + + - The ID in Kafka Advertised Listener Pattern + - The Endpoint Service Name + - The Bootstrap Ports + +- If your Apache Kafka service is hosted in Google Cloud, follow [Set Up Self-Hosted Kafka Private Service Connect in Google Cloud](/tidb-cloud/setup-self-hosted-kafka-private-service-connect.md) to ensure that the network connection is properly configured. After setup, provide the following information in the TiDB Cloud console to create the changefeed: + + - The ID in Kafka Advertised Listener Pattern + - The Service Attachment + - The Bootstrap Ports + +
    +
    If your Apache Kafka service is in an AWS VPC that has no internet access, take the following steps: @@ -39,7 +76,7 @@ If your Apache Kafka service is in an AWS VPC that has no internet access, take 3. If the Apache Kafka URL contains hostnames, you need to allow TiDB Cloud to be able to resolve the DNS hostnames of the Apache Kafka brokers. - 1. Follow the steps in [Enable DNS resolution for a VPC peering connection](https://docs.aws.amazon.com/vpc/latest/peering/modify-peering-connections.html#vpc-peering-dns). + 1. Follow the steps in [Enable DNS resolution for a VPC peering connection](https://docs.aws.amazon.com/vpc/latest/peering/vpc-peering-dns.html). 2. Enable the **Accepter DNS resolution** option. If your Apache Kafka service is in a Google Cloud VPC that has no internet access, take the following steps: @@ -49,6 +86,16 @@ If your Apache Kafka service is in a Google Cloud VPC that has no internet acces You must add the CIDR of the region where your TiDB Cloud cluster is located to the ingress firewall rules. The CIDR can be found on the **VPC Peering** page. Doing so allows the traffic to flow from your TiDB cluster to the Kafka brokers. +
    +
    + +If you want to provide Public IP access to your Apache Kafka service, assign Public IP addresses to all your Kafka brokers. + +It is **NOT** recommended to use Public IP in a production environment. + +
    +
    + ### Kafka ACL authorization To allow TiDB Cloud changefeeds to stream data to Apache Kafka and create Kafka topics automatically, ensure that the following permissions are added in Kafka: @@ -60,82 +107,147 @@ For example, if your Kafka cluster is in Confluent Cloud, you can see [Resources ## Step 1. Open the changefeed page for Apache Kafka -1. In the [TiDB Cloud console](https://tidbcloud.com), navigate to the cluster overview page of the target TiDB cluster, and then click **Changefeed** in the left navigation pane. -2. Click **Create Changefeed**, and select **Kafka** as **Target Type**. +1. Log in to the [TiDB Cloud console](https://tidbcloud.com). +2. Navigate to the cluster overview page of the target TiDB cluster, and then click **Changefeed** in the left navigation pane. +3. Click **Create Changefeed**, and select **Kafka** as **Target Type**. ## Step 2. Configure the changefeed target -1. Under **Brokers Configuration**, fill in your Kafka brokers endpoints. You can use commas `,` to separate multiple endpoints. -2. Select your Kafka version. If you do not know that, use Kafka V2. -3. Select a desired compression type for the data in this changefeed. -4. Enable the **TLS Encryption** option if your Kafka has enabled TLS encryption and you want to use TLS encryption for the Kafka connection. -5. Select the **Authentication** option according to your Kafka authentication configuration. +The steps vary depending on the connectivity method you select. + + +
    - - If your Kafka does not require authentication, keep the default option **DISABLE**. - - If your Kafka requires authentication, select the corresponding authentication type, and then fill in the user name and password of your Kafka account for authentication. +1. In **Connectivity Method**, select **VPC Peering** or **Public IP**, fill in your Kafka brokers endpoints. You can use commas `,` to separate multiple endpoints. +2. Select an **Authentication** option according to your Kafka authentication configuration. -6. Click **Next** to check the configurations you set and go to the next page. + - If your Kafka does not require authentication, keep the default option **Disable**. + - If your Kafka requires authentication, select the corresponding authentication type, and then fill in the **user name** and **password** of your Kafka account for authentication. + +3. Select your **Kafka Version**. If you do not know which one to use, use **Kafka v2**. +4. Select a **Compression** type for the data in this changefeed. +5. Enable the **TLS Encryption** option if your Kafka has enabled TLS encryption and you want to use TLS encryption for the Kafka connection. +6. Click **Next** to test the network connection. If the test succeeds, you will be directed to the next page. + +
    +
    + +1. In **Connectivity Method**, select **Private Link**. +2. Authorize the [AWS Principal](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_principal.html#principal-accounts) of TiDB Cloud to create an endpoint for your endpoint service. The AWS Principal is provided in the tip on the web page. +3. Make sure you select the same **Number of AZs** and **AZ IDs of Kafka Deployment**, and fill the same unique ID in **Kafka Advertised Listener Pattern** when you [Set Up Self-Hosted Kafka Private Link Service in AWS](/tidb-cloud/setup-self-hosted-kafka-private-link-service.md) in the **Network** section. +4. Fill in the **Endpoint Service Name** which is configured in [Set Up Self-Hosted Kafka Private Link Service in AWS](/tidb-cloud/setup-self-hosted-kafka-private-link-service.md). +5. Fill in the **Bootstrap Ports**. It is recommended that you set at least one port for one AZ. You can use commas `,` to separate multiple ports. +6. Select an **Authentication** option according to your Kafka authentication configuration. + + - If your Kafka does not require authentication, keep the default option **Disable**. + - If your Kafka requires authentication, select the corresponding authentication type, and then fill in the **user name** and **password** of your Kafka account for authentication. + +7. Select your **Kafka Version**. If you do not know which one to use, use **Kafka v2**. +8. Select a **Compression** type for the data in this changefeed. +9. Enable the **TLS Encryption** option if your Kafka has enabled TLS encryption and you want to use TLS encryption for the Kafka connection. +10. Click **Next** to test the network connection. If the test succeeds, you will be directed to the next page. +11. TiDB Cloud creates the endpoint for **Private Link**, which might take several minutes. +12. Once the endpoint is created, log in to your cloud provider console and accept the connection request. +13. Return to the [TiDB Cloud console](https://tidbcloud.com) to confirm that you have accepted the connection request. TiDB Cloud will test the connection and proceed to the next page if the test succeeds. + +
    +
    + +1. In **Connectivity Method**, select **Private Service Connect**. +2. Ensure that you fill in the same unique ID in **Kafka Advertised Listener Pattern** when you [Set Up Self-Hosted Kafka Private Service Connect in Google Cloud](/tidb-cloud/setup-self-hosted-kafka-private-service-connect.md) in the **Network** section. +3. Fill in the **Service Attachment** that you have configured in [Setup Self Hosted Kafka Private Service Connect in Google Cloud](/tidb-cloud/setup-self-hosted-kafka-private-service-connect.md) +4. Fill in the **Bootstrap Ports**. It is recommended that you provide more than one port. You can use commas `,` to separate multiple ports. +5. Select an **Authentication** option according to your Kafka authentication configuration. + + - If your Kafka does not require authentication, keep the default option **Disable**. + - If your Kafka requires authentication, select the corresponding authentication type, and then fill in the **user name** and **password** of your Kafka account for authentication. + +6. Select your **Kafka Version**. If you do not know which one to use, use **Kafka v2**. +7. Select a **Compression** type for the data in this changefeed. +8. Enable the **TLS Encryption** option if your Kafka has enabled TLS encryption and you want to use TLS encryption for the Kafka connection. +9. Click **Next** to test the network connection. If the test succeeds, you will be directed to the next page. +10. TiDB Cloud creates the endpoint for **Private Service Connect**, which might take several minutes. +11. Once the endpoint is created, log in to your cloud provider console and accept the connection request. +12. Return to the [TiDB Cloud console](https://tidbcloud.com) to confirm that you have accepted the connection request. TiDB Cloud will test the connection and proceed to the next page if the test succeeds. + +
    +
    ## Step 3. Set the changefeed 1. Customize **Table Filter** to filter the tables that you want to replicate. For the rule syntax, refer to [table filter rules](/table-filter.md). - - **Add filter rules**: you can set filter rules in this column. By default, there is a rule `*.*`, which stands for replicating all tables. When you add a new rule, TiDB Cloud queries all the tables in TiDB and displays only the tables that match the rules in the **Tables to be replicated** column. - - **Tables to be replicated**: this column shows the tables to be replicated. But it does not show the new tables to be replicated in the future or the schemas to be fully replicated. - - **Tables without valid keys**: this column shows tables without unique and primary keys. For these tables, because no unique identifier can be used by the downstream system to handle duplicate events, their data might be inconsistent during replication. To avoid such issues, it is recommended that you add unique keys or primary keys to these tables before the replication, or set filter rules to filter out these tables. For example, you can filter out the table `test.tbl1` using "!test.tbl1". + - **Filter Rules**: you can set filter rules in this column. By default, there is a rule `*.*`, which stands for replicating all tables. When you add a new rule, TiDB Cloud queries all the tables in TiDB and displays only the tables that match the rules in the box on the right. You can add up to 100 filter rules. + - **Tables with valid keys**: this column displays the tables that have valid keys, including primary keys or unique indexes. + - **Tables without valid keys**: this column shows tables that lack primary keys or unique keys. These tables present a challenge during replication because the absence of a unique identifier can result in inconsistent data when the downstream handles duplicate events. To ensure data consistency, it is recommended to add unique keys or primary keys to these tables before initiating the replication. Alternatively, you can add filter rules to exclude these tables. For example, you can exclude the table `test.tbl1` by using the rule `"!test.tbl1"`. + +2. Customize **Event Filter** to filter the events that you want to replicate. + + - **Tables matching**: you can set which tables the event filter will be applied to in this column. The rule syntax is the same as that used for the preceding **Table Filter** area. You can add up to 10 event filter rules per changefeed. + - **Ignored events**: you can set which types of events the event filter will exclude from the changefeed. -2. In the **Data Format** area, select your desired format of Kafka messages. +3. In the **Data Format** area, select your desired format of Kafka messages. - - Avro is a compact, fast, and binary data format with rich data structures, which is widely used in various flow systems. For more information, see [Avro data format](https://docs.pingcap.com/tidb/stable/ticdc-avro-protocol). - - Canal-JSON is a plain JSON text format, which is easy to parse. For more information, see [Canal-JSON data format](https://docs.pingcap.com/tidb/stable/ticdc-canal-json). + - Avro is a compact, fast, and binary data format with rich data structures, which is widely used in various flow systems. For more information, see [Avro data format](https://docs.pingcap.com/tidb/stable/ticdc-avro-protocol). + - Canal-JSON is a plain JSON text format, which is easy to parse. For more information, see [Canal-JSON data format](https://docs.pingcap.com/tidb/stable/ticdc-canal-json). + - Open Protocol is a row-level data change notification protocol that provides data sources for monitoring, caching, full-text indexing, analysis engines, and primary-secondary replication between different databases. For more information, see [Open Protocol data format](https://docs.pingcap.com/tidb/stable/ticdc-open-protocol). + - Debezium is a tool for capturing database changes. It converts each captured database change into a message called an "event" and sends these events to Kafka. For more information, see [Debezium data format](https://docs.pingcap.com/tidb/stable/ticdc-debezium). -3. Enable the **TiDB Extension** option if you want to add TiDB-extension fields to the Kafka message body. +4. Enable the **TiDB Extension** option if you want to add TiDB-extension fields to the Kafka message body. For more information about TiDB-extension fields, see [TiDB extension fields in Avro data format](https://docs.pingcap.com/tidb/stable/ticdc-avro-protocol#tidb-extension-fields) and [TiDB extension fields in Canal-JSON data format](https://docs.pingcap.com/tidb/stable/ticdc-canal-json#tidb-extension-field). -4. If you select **Avro** as your data format, you will see some Avro-specific configurations on the page. You can fill in these configurations as follows: +5. If you select **Avro** as your data format, you will see some Avro-specific configurations on the page. You can fill in these configurations as follows: - In the **Decimal** and **Unsigned BigInt** configurations, specify how TiDB Cloud handles the decimal and unsigned bigint data types in Kafka messages. - In the **Schema Registry** area, fill in your schema registry endpoint. If you enable **HTTP Authentication**, the fields for user name and password are displayed and automatically filled in with your TiDB cluster endpoint and password. -5. In the **Topic Distribution** area, select a distribution mode, and then fill in the topic name configurations according to the mode. +6. In the **Topic Distribution** area, select a distribution mode, and then fill in the topic name configurations according to the mode. If you select **Avro** as your data format, you can only choose the **Distribute changelogs by table to Kafka Topics** mode in the **Distribution Mode** drop-down list. The distribution mode controls how the changefeed creates Kafka topics, by table, by database, or creating one topic for all changelogs. - - **Distribute changelogs by table to Kafka Topics** + - **Distribute changelogs by table to Kafka Topics** If you want the changefeed to create a dedicated Kafka topic for each table, choose this mode. Then, all Kafka messages of a table are sent to a dedicated Kafka topic. You can customize topic names for tables by setting a topic prefix, a separator between a database name and table name, and a suffix. For example, if you set the separator as `_`, the topic names are in the format of `_`. For changelogs of non-row events, such as Create Schema Event, you can specify a topic name in the **Default Topic Name** field. The changefeed will create a topic accordingly to collect such changelogs. - - **Distribute changelogs by database to Kafka Topics** + - **Distribute changelogs by database to Kafka Topics** If you want the changefeed to create a dedicated Kafka topic for each database, choose this mode. Then, all Kafka messages of a database are sent to a dedicated Kafka topic. You can customize topic names of databases by setting a topic prefix and a suffix. For changelogs of non-row events, such as Resolved Ts Event, you can specify a topic name in the **Default Topic Name** field. The changefeed will create a topic accordingly to collect such changelogs. - - **Send all changelogs to one specified Kafka Topic** + - **Send all changelogs to one specified Kafka Topic** If you want the changefeed to create one Kafka topic for all changelogs, choose this mode. Then, all Kafka messages in the changefeed will be sent to one Kafka topic. You can define the topic name in the **Topic Name** field. -6. In the **Partition Distribution** area, you can decide which partition a Kafka message will be sent to: +7. In the **Partition Distribution** area, you can decide which partition a Kafka message will be sent to. You can define **a single partition dispatcher for all tables**, or **different partition dispatchers for different tables**. TiDB Cloud provides four types of dispatchers: - - **Distribute changelogs by index value to Kafka partition** + - **Distribute changelogs by primary key or index value to Kafka partition** - If you want the changefeed to send Kafka messages of a table to different partitions, choose this distribution method. The index value of a row changelog will determine which partition the changelog is sent to. This distribution method provides a better partition balance and ensures row-level orderliness. + If you want the changefeed to send Kafka messages of a table to different partitions, choose this distribution method. The primary key or index value of a row changelog will determine which partition the changelog is sent to. This distribution method provides a better partition balance and ensures row-level orderliness. - - **Distribute changelogs by table to Kafka partition** + - **Distribute changelogs by table to Kafka partition** If you want the changefeed to send Kafka messages of a table to one Kafka partition, choose this distribution method. The table name of a row changelog will determine which partition the changelog is sent to. This distribution method ensures table orderliness but might cause unbalanced partitions. -7. In the **Topic Configuration** area, configure the following numbers. The changefeed will automatically create the Kafka topics according to the numbers. + - **Distribute changelogs by timestamp to Kafka partition** + + If you want the changefeed to send Kafka messages to different Kafka partitions randomly, choose this distribution method. The commitTs of a row changelog will determine which partition the changelog is sent to. This distribution method provides a better partition balance and ensures orderliness in each partition. However, multiple changes of a data item might be sent to different partitions and the consumer progress of different consumers might be different, which might cause data inconsistency. Therefore, the consumer needs to sort the data from multiple partitions by commitTs before consuming. + + - **Distribute changelogs by column value to Kafka partition** + + If you want the changefeed to send Kafka messages of a table to different partitions, choose this distribution method. The specified column values of a row changelog will determine which partition the changelog is sent to. This distribution method ensures orderliness in each partition and guarantees that the changelog with the same column values is send to the same partition. + +8. In the **Topic Configuration** area, configure the following numbers. The changefeed will automatically create the Kafka topics according to the numbers. - - **Replication Factor**: controls how many Kafka servers each Kafka message is replicated to. - - **Partition Number**: controls how many partitions exist in a topic. + - **Replication Factor**: controls how many Kafka servers each Kafka message is replicated to. The valid value ranges from [`min.insync.replicas`](https://kafka.apache.org/33/documentation.html#brokerconfigs_min.insync.replicas) to the number of Kafka brokers. + - **Partition Number**: controls how many partitions exist in a topic. The valid value range is `[1, 10 * the number of Kafka brokers]`. -8. Click **Next**. +9. Click **Next**. ## Step 4. Configure your changefeed specification diff --git a/tidb-cloud/changefeed-sink-to-cloud-storage.md b/tidb-cloud/changefeed-sink-to-cloud-storage.md index cd1577a4b9555..596a2df4db24b 100644 --- a/tidb-cloud/changefeed-sink-to-cloud-storage.md +++ b/tidb-cloud/changefeed-sink-to-cloud-storage.md @@ -1,6 +1,6 @@ --- title: Sink to Cloud Storage -Summary: Learn how to create a changefeed to stream data from a TiDB Dedicated cluster to cloud storage, such as Amazon S3 and GCS. +summary: This document explains how to create a changefeed to stream data from TiDB Cloud to Amazon S3 or GCS. It includes restrictions, configuration steps for the destination, replication, and specification, as well as starting the replication process. --- # Sink to Cloud Storage @@ -9,12 +9,12 @@ This document describes how to create a changefeed to stream data from TiDB Clou > **Note:** > -> - To stream data to cloud storage, make sure that your TiDB cluster version is v7.1.1 or later. To upgrade your TiDB Dedicated cluster to v7.1.1 or later, [contact TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). -> - For [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters, the changefeed feature is unavailable. +> - To stream data to cloud storage, make sure that your TiDB cluster version is v7.1.1 or later. To upgrade your TiDB Cloud Dedicated cluster to v7.1.1 or later, [contact TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). +> - For [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters, the changefeed feature is unavailable. ## Restrictions -- For each TiDB Cloud cluster, you can create up to 5 changefeeds. +- For each TiDB Cloud cluster, you can create up to 100 changefeeds. - Because TiDB Cloud uses TiCDC to establish changefeeds, it has the same [restrictions as TiCDC](https://docs.pingcap.com/tidb/stable/ticdc-overview#unsupported-scenarios). - If the table to be replicated does not have a primary key or a non-null unique index, the absence of a unique constraint during replication could result in duplicated data being inserted downstream in some retry scenarios. @@ -45,8 +45,8 @@ For **GCS**, before filling **GCS Endpoint**, you need to first grant the GCS bu ![Create a role](/media/tidb-cloud/changefeed/sink-to-cloud-storage-gcs-create-role.png) - 3. Enter a name, description, ID, and role launch stage for the role. The role name cannot be changed after the role is created. - 4. Click **Add permissions**. Add the following read-only permissions to the role, and then click **Add**. + 3. Enter a name, description, ID, and role launch stage for the role. The role name cannot be changed after the role is created. + 4. Click **Add permissions**. Add the following permissions to the role, and then click **Add**. - storage.buckets.get - storage.objects.create @@ -78,7 +78,7 @@ For **GCS**, before filling **GCS Endpoint**, you need to first grant the GCS bu ![Get bucket URI](/media/tidb-cloud/changefeed/sink-to-cloud-storage-gcs-uri01.png) - - To get a folder's gsutil URI, open the folder, click the copy button, and add `gs://` as a prefix. For example, if the bucket name is `test-sink-gcs` and the folder name is `changefeed-xxx`, the URI would be `gs://test-sink-gcs/changefeed-xxx`. + - To get a folder's gsutil URI, open the folder, click the copy button, and add `gs://` as a prefix. For example, if the bucket name is `test-sink-gcs` and the folder name is `changefeed-xxx`, the URI would be `gs://test-sink-gcs/changefeed-xxx/`. ![Get bucket URI](/media/tidb-cloud/changefeed/sink-to-cloud-storage-gcs-uri02.png) @@ -87,7 +87,7 @@ For **GCS**, before filling **GCS Endpoint**, you need to first grant the GCS bu
    -Click **Next** to establish the connection from the TiDB Dedicated cluster to Amazon S3 or GCS. TiDB Cloud will automatically test and verify if the connection is successful. +Click **Next** to establish the connection from the TiDB Cloud Dedicated cluster to Amazon S3 or GCS. TiDB Cloud will automatically test and verify if the connection is successful. - If yes, you are directed to the next step of configuration. - If not, a connectivity error is displayed, and you need to handle the error. After the error is resolved, click **Next** to retry the connection. @@ -98,17 +98,22 @@ Click **Next** to establish the connection from the TiDB Dedicated cluster to Am ![the table filter of changefeed](/media/tidb-cloud/changefeed/sink-to-s3-02-table-filter.jpg) - - **Filter Rules**: you can set filter rules in this column. By default, there is a rule `*.*`, which stands for replicating all tables. When you add a new rule, TiDB Cloud queries all the tables in TiDB and displays only the tables that match the rules in the box on the right. + - **Filter Rules**: you can set filter rules in this column. By default, there is a rule `*.*`, which stands for replicating all tables. When you add a new rule, TiDB Cloud queries all the tables in TiDB and displays only the tables that match the rules in the box on the right. You can add up to 100 filter rules. - **Tables with valid keys**: this column displays the tables that have valid keys, including primary keys or unique indexes. - **Tables without valid keys**: this column shows tables that lack primary keys or unique keys. These tables present a challenge during replication because the absence of a unique identifier can result in inconsistent data when handling duplicate events downstream. To ensure data consistency, it is recommended to add unique keys or primary keys to these tables before initiating the replication. Alternatively, you can employ filter rules to exclude these tables. For example, you can exclude the table `test.tbl1` by using the rule `"!test.tbl1"`. -2. In the **Start Replication Position** area, select one of the following replication positions: +2. Customize **Event Filter** to filter the events that you want to replicate. + + - **Tables matching**: you can set which tables the event filter will be applied to in this column. The rule syntax is the same as that used for the preceding **Table Filter** area. You can add up to 10 event filter rules per changefeed. + - **Ignored events**: you can set which types of events the event filter will exclude from the changefeed. + +3. In the **Start Replication Position** area, select one of the following replication positions: - Start replication from now on - Start replication from a specific [TSO](https://docs.pingcap.com/tidb/stable/glossary#tso) - Start replication from a specific time -3. In the **Data Format** area, select either the **CSV** or **Canal-JSON** format. +4. In the **Data Format** area, select either the **CSV** or **Canal-JSON** format.
    @@ -133,7 +138,7 @@ Click **Next** to establish the connection from the TiDB Dedicated cluster to Am
    -4. In the **Flush Parameters** area, you can configure two items: +5. In the **Flush Parameters** area, you can configure two items: - **Flush Interval**: set to 60 seconds by default, adjustable within a range of 2 seconds to 10 minutes; - **File Size**: set to 64 MB by default, adjustable within a range of 1 MB to 512 MB. diff --git a/tidb-cloud/changefeed-sink-to-mysql.md b/tidb-cloud/changefeed-sink-to-mysql.md index bc3c285115a56..f0c13cc51f99c 100644 --- a/tidb-cloud/changefeed-sink-to-mysql.md +++ b/tidb-cloud/changefeed-sink-to-mysql.md @@ -1,6 +1,6 @@ --- title: Sink to MySQL -Summary: Learn how to create a changefeed to stream data from TiDB Cloud to MySQL. +summary: This document explains how to stream data from TiDB Cloud to MySQL using the Sink to MySQL changefeed. It includes restrictions, prerequisites, and steps to create a MySQL sink for data replication. The process involves setting up network connections, loading existing data to MySQL, and creating target tables in MySQL. After completing the prerequisites, users can create a MySQL sink to replicate data to MySQL. --- # Sink to MySQL @@ -9,12 +9,12 @@ This document describes how to stream data from TiDB Cloud to MySQL using the ** > **Note:** > -> - To use the changefeed feature, make sure that your TiDB Dedicated cluster version is v6.4.0 or later. -> - For [TiDB Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-serverless), the changefeed feature is unavailable. +> - To use the changefeed feature, make sure that your TiDB Cloud Dedicated cluster version is v6.1.3 or later. +> - For [TiDB Cloud Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless), the changefeed feature is unavailable. ## Restrictions -- For each TiDB Cloud cluster, you can create up to 5 changefeeds. +- For each TiDB Cloud cluster, you can create up to 100 changefeeds. - Because TiDB Cloud uses TiCDC to establish changefeeds, it has the same [restrictions as TiCDC](https://docs.pingcap.com/tidb/stable/ticdc-overview#unsupported-scenarios). - If the table to be replicated does not have a primary key or a non-null unique index, the absence of a unique constraint during replication could result in duplicated data being inserted downstream in some retry scenarios. @@ -35,7 +35,7 @@ If your MySQL service is in an AWS VPC that has no public internet access, take 1. [Set up a VPC peering connection](/tidb-cloud/set-up-vpc-peering-connections.md) between the VPC of the MySQL service and your TiDB cluster. 2. Modify the inbound rules of the security group that the MySQL service is associated with. - You must add [the CIDR of the region where your TiDB Cloud cluster is located](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-project-cidr) to the inbound rules. Doing so allows the traffic to flow from your TiDB Cluster to the MySQL instance. + You must add [the CIDR of the region where your TiDB Cloud cluster is located](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-cidr-for-a-region) to the inbound rules. Doing so allows the traffic to flow from your TiDB Cluster to the MySQL instance. 3. If the MySQL URL contains a hostname, you need to allow TiDB Cloud to be able to resolve the DNS hostname of the MySQL service. @@ -48,7 +48,7 @@ If your MySQL service is in a Google Cloud VPC that has no public internet acces 2. [Set up a VPC peering connection](/tidb-cloud/set-up-vpc-peering-connections.md) between the VPC of the MySQL service and your TiDB cluster. 3. Modify the ingress firewall rules of the VPC where MySQL is located. - You must add [the CIDR of the region where your TiDB Cloud cluster is located](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-project-cidr) to the ingress firewall rules. Doing so allows the traffic to flow from your TiDB Cluster to the MySQL endpoint. + You must add [the CIDR of the region where your TiDB Cloud cluster is located](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-cidr-for-a-region) to the ingress firewall rules. Doing so allows the traffic to flow from your TiDB Cluster to the MySQL endpoint. ### Load existing data (optional) @@ -69,7 +69,7 @@ To load the existing data: SET GLOBAL tidb_gc_life_time = '720h'; ``` -2. Use [Dumpling](https://docs.pingcap.com/tidb/stable/dumpling-overview) to export data from your TiDB cluster, then use community tools such as myloader to load data to the MySQL service. +2. Use [Dumpling](https://docs.pingcap.com/tidb/stable/dumpling-overview) to export data from your TiDB cluster, then use community tools such as [mydumper/myloader](https://centminmod.com/mydumper.html) to load data to the MySQL service. 3. From the [exported files of Dumpling](https://docs.pingcap.com/tidb/stable/dumpling-overview#format-of-exported-files), get the start position of MySQL sink from the metadata file: @@ -104,32 +104,37 @@ After completing the prerequisites, you can sink your data to MySQL. 5. Customize **Table Filter** to filter the tables that you want to replicate. For the rule syntax, refer to [table filter rules](/table-filter.md). - - **Add filter rules**: you can set filter rules in this column. By default, there is a rule `*. *`, which stands for replicating all tables. When you add a new rule, TiDB Cloud queries all the tables in TiDB and displays only the tables that match the rules in the box on the right. - - **Tables to be replicated**: this column shows the tables to be replicated. But it does not show the new tables to be replicated in the future or the schemas to be fully replicated. - - **Tables without valid keys**: this column shows tables without unique and primary keys. For these tables, because no unique identifier can be used by the downstream system to handle duplicate events, their data might be inconsistent during replication. To avoid such issues, it is recommended that you add unique keys or primary keys to these tables before the replication, or set filter rules to filter out these tables. For example, you can filter out the table `test.tbl1` using "!test.tbl1". + - **Filter Rules**: you can set filter rules in this column. By default, there is a rule `*.*`, which stands for replicating all tables. When you add a new rule, TiDB Cloud queries all the tables in TiDB and displays only the tables that match the rules in the box on the right. You can add up to 100 filter rules. + - **Tables with valid keys**: this column displays the tables that have valid keys, including primary keys or unique indexes. + - **Tables without valid keys**: this column shows tables that lack primary keys or unique keys. These tables present a challenge during replication because the absence of a unique identifier can result in inconsistent data when the downstream handles duplicate events. To ensure data consistency, it is recommended to add unique keys or primary keys to these tables before initiating the replication. Alternatively, you can add filter rules to exclude these tables. For example, you can exclude the table `test.tbl1` by using the rule `"!test.tbl1"`. -6. In **Start Position**, configure the starting position for your MySQL sink. +6. Customize **Event Filter** to filter the events that you want to replicate. + + - **Tables matching**: you can set which tables the event filter will be applied to in this column. The rule syntax is the same as that used for the preceding **Table Filter** area. You can add up to 10 event filter rules per changefeed. + - **Ignored events**: you can set which types of events the event filter will exclude from the changefeed. + +7. In **Start Replication Position**, configure the starting position for your MySQL sink. - If you have [loaded the existing data](#load-existing-data-optional) using Dumpling, select **Start replication from a specific TSO** and fill in the TSO that you get from Dumpling exported metadata files. - If you do not have any data in the upstream TiDB cluster, select **Start replication from now on**. - Otherwise, you can customize the start time point by choosing **Start replication from a specific time**. -7. Click **Next** to configure your changefeed specification. +8. Click **Next** to configure your changefeed specification. - In the **Changefeed Specification** area, specify the number of Replication Capacity Units (RCUs) to be used by the changefeed. - In the **Changefeed Name** area, specify a name for the changefeed. -8. Click **Next** to review the changefeed configuration. +9. Click **Next** to review the changefeed configuration. - If you confirm all configurations are correct, check the compliance of cross-region replication, and click **Create**. + If you confirm that all configurations are correct, check the compliance of cross-region replication, and click **Create**. If you want to modify some configurations, click **Previous** to go back to the previous configuration page. -9. The sink starts soon, and you can see the status of the sink changes from "**Creating**" to "**Running**". +10. The sink starts soon, and you can see the status of the sink changes from **Creating** to **Running**. Click the changefeed name, and you can see more details about the changefeed, such as the checkpoint, replication latency, and other metrics. -10. If you have [loaded the existing data](#load-existing-data-optional) using Dumpling, you need to restore the GC time to its original value (the default value is `10m`) after the sink is created: +11. If you have [loaded the existing data](#load-existing-data-optional) using Dumpling, you need to restore the GC time to its original value (the default value is `10m`) after the sink is created: {{< copyable "sql" >}} diff --git a/tidb-cloud/changefeed-sink-to-tidb-cloud.md b/tidb-cloud/changefeed-sink-to-tidb-cloud.md index e4cc769a8ba84..d2c658622d030 100644 --- a/tidb-cloud/changefeed-sink-to-tidb-cloud.md +++ b/tidb-cloud/changefeed-sink-to-tidb-cloud.md @@ -1,36 +1,36 @@ --- title: Sink to TiDB Cloud -Summary: Learn how to create a changefeed to stream data from a TiDB Dedicated cluster to a TiDB Serverless cluster. +summary: This document explains how to stream data from a TiDB Cloud Dedicated cluster to a TiDB Cloud Serverless cluster. There are restrictions on the number of changefeeds and regions available for the feature. Prerequisites include extending tidb_gc_life_time, backing up data, and obtaining the start position of TiDB Cloud sink. To create a TiDB Cloud sink, navigate to the cluster overview page, establish the connection, customize table and event filters, fill in the start replication position, specify the changefeed specification, review the configuration, and create the sink. Finally, restore tidb_gc_life_time to its original value. --- # Sink to TiDB Cloud -This document describes how to stream data from a TiDB Dedicated cluster to a TiDB Serverless cluster. +This document describes how to stream data from a TiDB Cloud Dedicated cluster to a TiDB Cloud Serverless cluster. > **Note:** > -> To use the Changefeed feature, make sure that your TiDB Dedicated cluster version is v6.4.0 or later. +> To use the Changefeed feature, make sure that your TiDB Cloud Dedicated cluster version is v6.1.3 or later. ## Restrictions -- For each TiDB Cloud cluster, you can create up to 5 changefeeds. +- For each TiDB Cloud cluster, you can create up to 100 changefeeds. - Because TiDB Cloud uses TiCDC to establish changefeeds, it has the same [restrictions as TiCDC](https://docs.pingcap.com/tidb/stable/ticdc-overview#unsupported-scenarios). - If the table to be replicated does not have a primary key or a non-null unique index, the absence of a unique constraint during replication could result in duplicated data being inserted downstream in some retry scenarios. -- The **Sink to TiDB Cloud** feature is only available to TiDB Dedicated clusters that are in the following AWS regions and created after November 9, 2022: +- The **Sink to TiDB Cloud** feature is only available to TiDB Cloud Dedicated clusters that are in the following AWS regions and created after November 9, 2022: - AWS Oregon (us-west-2) - AWS Frankfurt (eu-central-1) - AWS Singapore (ap-southeast-1) - AWS Tokyo (ap-northeast-1) -- The source TiDB Dedicated cluster and the destination TiDB Serverless cluster must be in the same project and the same region. -- The **Sink to TiDB Cloud** feature only supports network connection via private endpoints. When you create a changefeed to stream data from a TiDB Dedicated cluster to a TiDB Serverless cluster, TiDB Cloud will automatically set up the private endpoint connection between the two clusters. +- The source TiDB Cloud Dedicated cluster and the destination TiDB Cloud Serverless cluster must be in the same project and the same region. +- The **Sink to TiDB Cloud** feature only supports network connection via private endpoints. When you create a changefeed to stream data from a TiDB Cloud Dedicated cluster to a TiDB Cloud Serverless cluster, TiDB Cloud will automatically set up the private endpoint connection between the two clusters. ## Prerequisites -The **Sink to TiDB Cloud** connector can only sink incremental data from a TiDB Dedicated cluster to a TiDB Serverless cluster after a certain [TSO](https://docs.pingcap.com/tidb/stable/glossary#tso). +The **Sink to TiDB Cloud** connector can only sink incremental data from a TiDB Cloud Dedicated cluster to a TiDB Cloud Serverless cluster after a certain [TSO](https://docs.pingcap.com/tidb/stable/glossary#tso). -Before creating a changefeed, you need to export existing data from the source TiDB Dedicated cluster and load the data to the destination TiDB Serverless cluster. +Before creating a changefeed, you need to export existing data from the source TiDB Cloud Dedicated cluster and load the data to the destination TiDB Cloud Serverless cluster. 1. Extend the [tidb_gc_life_time](https://docs.pingcap.com/tidb/stable/system-variables#tidb_gc_life_time-new-in-v50) to be longer than the total time of the following two operations, so that historical data during the time is not garbage collected by TiDB. @@ -43,7 +43,7 @@ Before creating a changefeed, you need to export existing data from the source T SET GLOBAL tidb_gc_life_time = '720h'; ``` -2. [Back up data](/tidb-cloud/backup-and-restore.md#backup) from your TiDB Dedicated cluster, then use community tools such as [mydumper/myloader](https://centminmod.com/mydumper.html) to load data to the destination TiDB Serverless cluster. +2. [Back up data](/tidb-cloud/backup-and-restore.md#backup) from your TiDB Cloud Dedicated cluster, then use community tools such as [mydumper/myloader](https://centminmod.com/mydumper.html) to load data to the destination TiDB Cloud Serverless cluster. 3. From the [exported files of Dumpling](https://docs.pingcap.com/tidb/stable/dumpling-overview#format-of-exported-files), get the start position of TiDB Cloud sink from the metadata file: @@ -59,13 +59,13 @@ Before creating a changefeed, you need to export existing data from the source T ## Create a TiDB Cloud sink -After completing the prerequisites, you can sink your data to the destination TiDB Serverless cluster. +After completing the prerequisites, you can sink your data to the destination TiDB Cloud Serverless cluster. 1. Navigate to the cluster overview page of the target TiDB cluster, and then click **Changefeed** in the left navigation pane. 2. Click **Create Changefeed**, and select **TiDB Cloud** as the destination. -3. In the **TiDB Cloud Connection** area, select the destination TiDB Serverless cluster, and then fill in the user name and password of the destination cluster. +3. In the **TiDB Cloud Connection** area, select the destination TiDB Cloud Serverless cluster, and then fill in the user name and password of the destination cluster. 4. Click **Next** to establish the connection between the two TiDB clusters and test whether the changefeed can connect them successfully: @@ -74,28 +74,33 @@ After completing the prerequisites, you can sink your data to the destination Ti 5. Customize **Table Filter** to filter the tables that you want to replicate. For the rule syntax, refer to [table filter rules](/table-filter.md). - - **Filter rules**: you can set filter rules in this column. By default, there is a rule `*. *`, which stands for replicating all tables. When you add a new rule, TiDB Cloud queries all the tables in TiDB and displays only the tables that match the rules in the box on the right. - - **Tables to be replicated**: this column shows the tables to be replicated. But it does not show the new tables to be replicated in the future or the schemas to be fully replicated. - - **Tables without valid keys**: this column shows tables without unique and primary keys. For these tables, because no unique identifier can be used by the downstream system to handle duplicate events, their data might be inconsistent during replication. To avoid such issues, it is recommended that you add unique keys or primary keys to these tables before the replication, or set filter rules to filter out these tables. For example, you can filter out the table `test.tbl1` using "!test.tbl1". + - **Filter Rules**: you can set filter rules in this column. By default, there is a rule `*.*`, which stands for replicating all tables. When you add a new rule, TiDB Cloud queries all the tables in TiDB and displays only the tables that match the rules in the box on the right. You can add up to 100 filter rules. + - **Tables with valid keys**: this column displays the tables that have valid keys, including primary keys or unique indexes. + - **Tables without valid keys**: this column shows tables that lack primary keys or unique keys. These tables present a challenge during replication because the absence of a unique identifier can result in inconsistent data when the downstream handles duplicate events. To ensure data consistency, it is recommended to add unique keys or primary keys to these tables before initiating the replication. Alternatively, you can add filter rules to exclude these tables. For example, you can exclude the table `test.tbl1` by using the rule `"!test.tbl1"`. -6. In the **Start Replication Position** area, fill in the TSO that you get from Dumpling exported metadata files. +6. Customize **Event Filter** to filter the events that you want to replicate. -7. Click **Next** to configure your changefeed specification. + - **Tables matching**: you can set which tables the event filter will be applied to in this column. The rule syntax is the same as that used for the preceding **Table Filter** area. You can add up to 10 event filter rules per changefeed. + - **Ignored events**: you can set which types of events the event filter will exclude from the changefeed. + +7. In the **Start Replication Position** area, fill in the TSO that you get from Dumpling exported metadata files. + +8. Click **Next** to configure your changefeed specification. - In the **Changefeed Specification** area, specify the number of Replication Capacity Units (RCUs) to be used by the changefeed. - In the **Changefeed Name** area, specify a name for the changefeed. -8. Click **Next** to review the Changefeed configuration. +9. Click **Next** to review the changefeed configuration. If you confirm that all configurations are correct, check the compliance of cross-region replication, and click **Create**. If you want to modify some configurations, click **Previous** to go back to the previous configuration page. -9. The sink starts soon, and you can see the status of the sink changes from **Creating** to **Running**. +10. The sink starts soon, and you can see the status of the sink changes from **Creating** to **Running**. Click the changefeed name, and you can see more details about the changefeed, such as the checkpoint, replication latency, and other metrics. -10. Restore [tidb_gc_life_time](https://docs.pingcap.com/tidb/stable/system-variables#tidb_gc_life_time-new-in-v50) to its original value (the default value is `10m`) after the sink is created: +11. Restore [tidb_gc_life_time](https://docs.pingcap.com/tidb/stable/system-variables#tidb_gc_life_time-new-in-v50) to its original value (the default value is `10m`) after the sink is created: ```sql SET GLOBAL tidb_gc_life_time = '10m'; diff --git a/tidb-cloud/cli-reference.md b/tidb-cloud/cli-reference.md index 44d96bdde1cda..674e1871715e3 100644 --- a/tidb-cloud/cli-reference.md +++ b/tidb-cloud/cli-reference.md @@ -3,7 +3,11 @@ title: TiDB Cloud CLI Reference summary: Provides an overview of TiDB Cloud CLI. --- -# TiDB Cloud CLI Reference +# TiDB Cloud CLI Reference Beta + +> **Note:** +> +> TiDB Cloud CLI is in beta. TiDB Cloud CLI is a command line interface, which allows you to operate TiDB Cloud from your terminal with a few lines of commands. In the TiDB Cloud CLI, you can easily manage your TiDB Cloud clusters, import data to your clusters, and perform more operations. @@ -17,17 +21,20 @@ The following table lists the commands available for the TiDB Cloud CLI. To use the `ticloud` CLI in your terminal, run `ticloud [command] [subcommand]`. If you are using [TiUP](https://docs.pingcap.com/tidb/stable/tiup-overview), use `tiup cloud [command] [subcommand]` instead. -| Command | Subcommand | Description | -|------------|------------------------------------------------------------|----------------------------------------------------------------------------------------------------------| -| cluster | create, delete, describe, list, connect-info | Manage clusters | -| branch | create, delete, describe, list, connect-info | Manage branches | -| completion | bash, fish, powershell, zsh | Generate completion script for specified shell | -| config | create, delete, describe, edit, list, set, use | Configure user profiles | -| connect | - | Connect to a TiDB cluster | -| help | cluster, completion, config, help, import, project, update | View help for any command | -| import | cancel, describe, list, start | Manage [import](/tidb-cloud/tidb-cloud-migration-overview.md#import-data-from-files-to-tidb-cloud) tasks | -| project | list | Manage projects | -| update | - | Update the CLI to the latest version | +| Command | Subcommand | Description | +|-----------------------|-----------------------------------------------------------------------|------------------------------------------------| +| auth | login, logout, whoami | Login and logout | +| serverless (alias: s) | create, delete, describe, list, update, spending-limit, region, shell | Manage TiDB Cloud Serverless clusters | +| serverless branch | create, delete, describe, list, shell | Manage TiDB Cloud Serverless branches | +| serverless import | cancel, describe, list, start | Manage TiDB Cloud Serverless import tasks | +| serverless export | create, describe, list, cancel, download | Manage TiDB Cloud Serverless export tasks | +| serverless sql-user | create, list, delete, update | Manage TiDB Cloud Serverless SQL users | +| ai | - | Chat with TiDB Bot | +| completion | bash, fish, powershell, zsh | Generate completion script for specified shell | +| config | create, delete, describe, edit, list, set, use | Configure user profiles | +| project | list | Manage projects | +| upgrade | - | Update the CLI to the latest version | +| help | auth, config, serverless, ai, project, upgrade, help, completion | View help for any command | ## Command modes @@ -43,12 +50,20 @@ The TiDB Cloud CLI provides two modes for some commands for easy use: ## User profile -For the TiDB Cloud CLI, a user profile is a collection of properties associated with a user, including the profile name, public key, and private key. To use TiDB Cloud CLI, you must create a user profile first. +For the TiDB Cloud CLI, a user profile is a collection of properties associated with a user, including the profile name, public key, private key, and OAuth token. To use TiDB Cloud CLI, you must have a user profile. -### Create a user profile +### Create a user profile with TiDB Cloud API key Use [`ticloud config create`](/tidb-cloud/ticloud-config-create.md) to create a user profile. +### Create a user profile with OAuth token + +Use [`ticloud auth login`](/tidb-cloud/ticloud-auth-login.md) to assign OAuth token to the current profile. If no profiles exist, a profile named `default` will be created automatically. + +> **Note:** +> +> In the preceding two methods, the TiDB Cloud API key takes precedence over the OAuth token. If both are available in the current profile, the API key will be used. + ### List all user profiles Use [`ticloud config list`](/tidb-cloud/ticloud-config-list.md) to list all user profiles. @@ -103,10 +118,11 @@ Use [`ticloud config delete`](/tidb-cloud/ticloud-config-delete.md) to delete a The following table lists the global flags for the TiDB Cloud CLI. -| Flag | Description | Required | Note | -|----------------------|-----------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active user profile used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|---------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active user profile used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enable debug mode | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/config-s3-and-gcs-access.md b/tidb-cloud/config-s3-and-gcs-access.md index 641366334d176..fd438fef4d211 100644 --- a/tidb-cloud/config-s3-and-gcs-access.md +++ b/tidb-cloud/config-s3-and-gcs-access.md @@ -1,11 +1,13 @@ --- -title: Configure Amazon S3 Access and GCS Access +title: Configure External Storage Access for TiDB Cloud Dedicated summary: Learn how to configure Amazon Simple Storage Service (Amazon S3) access and Google Cloud Storage (GCS) access. --- -# Configure Amazon S3 Access and GCS Access +# Configure External Storage Access for TiDB Cloud Dedicated -If your source data is stored in Amazon S3 or Google Cloud Storage (GCS) buckets, before importing or migrating the data to TiDB Cloud, you need to configure cross-account access to the buckets. This document describes how to do this. +If your source data is stored in Amazon S3 or Google Cloud Storage (GCS) buckets, before importing or migrating the data to TiDB Cloud, you need to configure cross-account access to the buckets. This document describes how to do this for TiDB Cloud Dedicated clusters. + +If you need to configure these external storages for TiDB Cloud Serverless clusters, see [Configure External Storage Access for TiDB Cloud Serverless](/tidb-cloud/serverless-external-storage.md). ## Configure Amazon S3 access @@ -19,7 +21,7 @@ To allow TiDB Cloud to access the source data in your Amazon S3 bucket, you need Configure the bucket access for TiDB Cloud and get the Role ARN as follows: -1. In the [TiDB Cloud console](https://tidbcloud.com/), get the TiDB Cloud account ID and external ID of the target TiDB cluster. +1. In the [TiDB Cloud console](https://tidbcloud.com/), get the corresponding TiDB Cloud account ID and external ID of the target TiDB cluster. 1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. @@ -29,25 +31,29 @@ Configure the bucket access for TiDB Cloud and get the Role ARN as follows: 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. - 3. On the **Import** page, click **Import Data** in the upper-right corner and select **From S3**. + 3. On the **Import** page, select **Import data from S3**. + + If this is your first time importing data into this cluster, select **Import From Amazon S3**. - 4. On the **Import from S3** page, click **Guide for getting the required Role ARN** to get the TiDB Cloud Account ID and TiDB Cloud External ID. Take a note of these IDs for later use. + 4. On the **Import Data from Amazon S3** page, click the link under **Role ARN**. The **Add New Role ARN** dialog is displayed. + + 5. Expand **Create Role ARN manually** to get the TiDB Cloud Account ID and TiDB Cloud External ID. Take a note of these IDs for later use. 2. In the AWS Management Console, create a managed policy for your Amazon S3 bucket. - 1. Sign in to the AWS Management Console and open the Amazon S3 console at . + 1. Sign in to the AWS Management Console and open the Amazon S3 console at [https://console.aws.amazon.com/s3/](https://console.aws.amazon.com/s3/). 2. In the **Buckets** list, choose the name of your bucket with the source data, and then click **Copy ARN** to get your S3 bucket ARN (for example, `arn:aws:s3:::tidb-cloud-source-data`). Take a note of the bucket ARN for later use. ![Copy bucket ARN](/media/tidb-cloud/copy-bucket-arn.png) - 3. Open the IAM console at , click **Policies** in the navigation pane on the left, and then click **Create Policy**. + 3. Open the IAM console at [https://console.aws.amazon.com/iam/](https://console.aws.amazon.com/iam/), click **Policies** in the navigation pane on the left, and then click **Create Policy**. ![Create a policy](/media/tidb-cloud/aws-create-policy.png) 4. On the **Create policy** page, click the **JSON** tab. 5. Copy the following access policy template and paste it to the policy text field. - ``` + ```json { "Version": "2012-10-17", "Statement": [ @@ -83,13 +89,27 @@ Configure the bucket access for TiDB Cloud and get the Role ARN as follows: For example, `"Resource": "arn:aws:s3:::tidb-cloud-source-data"`. - 6. Click **Next: Tags**, add a tag of the policy (optional), and then click **Next:Review**. + - If you have enabled AWS Key Management Service key (SSE-KMS) with customer-managed key encryption, make sure the following configuration is included in the policy. `"arn:aws:kms:ap-northeast-1:105880447796:key/c3046e91-fdfc-4f3a-acff-00597dd3801f"` is a sample KMS key of the bucket. - 7. Set a policy name, and then click **Create policy**. + ``` + { + "Sid": "AllowKMSkey", + "Effect": "Allow", + "Action": [ + "kms:Decrypt" + ], + "Resource": "arn:aws:kms:ap-northeast-1:105880447796:key/c3046e91-fdfc-4f3a-acff-00597dd3801f" + } + ``` + + If the objects in your bucket have been copied from another encrypted bucket, the KMS key value needs to include the keys of both buckets. For example, `"Resource": ["arn:aws:kms:ap-northeast-1:105880447796:key/c3046e91-fdfc-4f3a-acff-00597dd3801f","arn:aws:kms:ap-northeast-1:495580073302:key/0d7926a7-6ecc-4bf7-a9c1-a38f0faec0cd"]`. + + 6. Click **Next**. + 7. Set a policy name, add a tag of the policy (optional), and then click **Create policy**. 3. In the AWS Management Console, create an access role for TiDB Cloud and get the role ARN. - 1. In the IAM console at , click **Roles** in the navigation pane on the left, and then click **Create role**. + 1. In the IAM console at [https://console.aws.amazon.com/iam/](https://console.aws.amazon.com/iam/), click **Roles** in the navigation pane on the left, and then click **Create role**. ![Create a role](/media/tidb-cloud/aws-create-role.png) @@ -97,7 +117,7 @@ Configure the bucket access for TiDB Cloud and get the Role ARN as follows: - Under **Trusted entity type**, select **AWS account**. - Under **An AWS account**, select **Another AWS account**, and then paste the TiDB Cloud account ID to the **Account ID** field. - - Under **Options**, click **Require external ID (Best practice when a third party will assume this role)**, and then paste the TiDB Cloud External ID to the **External ID** field. If the role is created without "Require external ID", once the configuration is done for one TiDB cluster in a project, all TiDB clusters in that project can use the same Role ARN to access your Amazon S3 bucket. If the role is created with the account ID and external ID, only the corresponding TiDB cluster can access the bucket. + - Under **Options**, click **Require external ID** to avoid the [confused deputy problem](https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html), and then paste the TiDB Cloud External ID to the **External ID** field. If the role is created without "Require external ID", anyone with your S3 bucket URI and IAM role ARN might be able to access your Amazon S3 bucket. If the role is created with both the account ID and external ID, only TiDB clusters running in the same project and the same region can access the bucket. 3. Click **Next** to open the policy list, choose the policy you just created, and then click **Next**. 4. Under **Role details**, set a name for the role, and then click **Create role** in the lower-right corner. After the role is created, the list of roles is displayed. @@ -148,7 +168,11 @@ To allow TiDB Cloud to access the source data in your GCS bucket, you need to co 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. - 3. Click **Import Data** in the upper-right corner, click **Show Google Cloud Service Account ID**, and then copy the Service Account ID for later use. + 3. Click **Import Data** in the upper-right corner. + + If this is your first time importing data into this cluster, select **Import From GCS**. + + 4. Click **Show Google Cloud Server Account ID**, and then copy the Service Account ID for later use. 2. In the Google Cloud console, create an IAM role for your GCS bucket. diff --git a/tidb-cloud/configure-ip-access-list.md b/tidb-cloud/configure-ip-access-list.md index afec8122dbf26..f08d894a6fa9b 100644 --- a/tidb-cloud/configure-ip-access-list.md +++ b/tidb-cloud/configure-ip-access-list.md @@ -1,58 +1,24 @@ --- title: Configure an IP Access List -summary: Learn how to configure IP addresses that are allowed to access your TiDB Dedicated cluster. +summary: Learn how to configure IP addresses that are allowed to access your TiDB Cloud Dedicated cluster. --- # Configure an IP Access List -For each TiDB Dedicated cluster in TiDB Cloud, you can configure an IP access list to filter internet traffic trying to access the cluster, which works similarly to a firewall access control list. After the configuration, only the clients and applications whose IP addresses are in the IP access list can connect to your TiDB Dedicated cluster. +For each TiDB Cloud Dedicated cluster in TiDB Cloud, you can configure an IP access list to filter internet traffic trying to access the cluster, which works similarly to a firewall access control list. After the configuration, only the clients and applications whose IP addresses are in the IP access list can connect to your TiDB Cloud Dedicated cluster. > **Note:** > -> Configuring the IP access list is only available for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. +> Configuring the IP access list is only available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters. -For a TiDB Dedicated cluster, you can configure its IP access list in either of the following ways: +To configure an IP access list, take the following steps in the [TiDB Cloud console](https://tidbcloud.com/console/clusters): -- [Configure an IP access list in standard connection](#configure-an-ip-access-list-in-standard-connection) +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. +2. In the left navigation pane, click **Networking**, and then click **Add IP Address**. +3. In the dialog, choose one of the following options: -- [Configure an IP access list in security settings](#configure-an-ip-access-list-in-security-settings) + - **Allow access from anywhere**: allows all IP addresses to access TiDB Cloud. This option exposes your cluster to the internet completely and is highly risky. + - **Use IP addresses** (recommended): you can add a list of IPs and CIDR addresses that are allowed to access TiDB Cloud via a SQL client. -## Configure an IP access list in standard connection - -To configure an IP access list for your TiDB Dedicated cluster in standard connection, take the following steps: - -1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. -2. In the row of your TiDB Dedicated cluster, click **...** and select **Connect**. A dialog is displayed. -3. In the dialog, locate **Step 1: Create traffic filter** on the **Standard Connection** tab and configure the IP access list. - - - If the IP access list of your cluster has not been set, you can click **Add My Current IP Address** to add your current IP address to the IP access list, and then click **Add Item** to add more IP addresses if necessary. Next, click **Update Filter** to save the configuration. - - > **Note:** - > - > For each TiDB Dedicated cluster, you can add up to 7 IP addresses to the IP access list. To apply for a quota to add more IP addresses, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). - - - If the IP access list of your cluster has been set, click **Edit** to add, edit, or remove IP addresses, and then click **Update Filter** to save the configuration. - - - To allow any IP address to access your cluster (not recommended), click **Allow Access From Anywhere**, and then click **Update Filter**. According to security best practices, it is NOT recommended that you allow any IP address to access your cluster, as this would expose your cluster to the internet completely, which is highly risky. - -## Configure an IP access list in security settings - -To configure an IP access list for your TiDB Dedicated cluster in security settings, take the following steps: - -1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. -2. In the row of your TiDB Dedicated cluster, click **...** and select **Security Settings**. A security setting dialog is displayed. -3. In the dialog, configure the IP access list as follows: - - - To add your current IP address to the IP access list, click **Add My Current IP Address**. - - - To add an IP address to the IP access list, enter the IP address and description, and click **Add to IP List**. - - > **Note:** - > - > For each TiDB Dedicated cluster, you can add up to 7 IP addresses to the IP access list. To apply for a quota to add more IP addresses, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). - - - To allow any IP address to access your cluster (not recommended), click **Allow Access From Anywhere**. According to security best practices, it is NOT recommended that you allow any IP address to access your cluster, as this would expose your cluster to the internet completely, which is highly risky. - - - To remove an IP address from the access list, click **Remove** in the line of the IP address. - -4. Click **Apply** to save the configuration. \ No newline at end of file +4. If you choose **Use IP addresses**, add IP addresses or CIDR range with an optional description. For each TiDB Cloud Dedicated cluster, you can add up to 100 IP addresses. +5. Click **Confirm** to save your changes. diff --git a/tidb-cloud/configure-maintenance-window.md b/tidb-cloud/configure-maintenance-window.md index 21e36c8b5c86b..c8403fc3459c6 100644 --- a/tidb-cloud/configure-maintenance-window.md +++ b/tidb-cloud/configure-maintenance-window.md @@ -7,13 +7,13 @@ summary: Learn how to configure maintenance window for your cluster. A maintenance window is a designated timeframe during which planned maintenance tasks, such as operating system updates, security patches, and infrastructure upgrades, are performed automatically to ensure the reliability, security, and performance of the TiDB Cloud service. -During a maintenance window, the maintenance is executed on TiDB Dedicated clusters one by one so the overall impact is minimal. Although there might be temporary connection disruptions or QPS fluctuations, the clusters remain available, and the existing data import, backup, restore, migration, and replication tasks can still run normally. +During a maintenance window, the maintenance is executed on TiDB Cloud Dedicated clusters one by one so the overall impact is minimal. Although there might be temporary connection disruptions or QPS fluctuations, the clusters remain available, and the existing data import, backup, restore, migration, and replication tasks can still run normally. By configuring the maintenance window, you can easily schedule and manage maintenance tasks to minimize the maintenance impact. For example, you can set the start time of the maintenance window to avoid peak hours of your application workloads. > **Note:** > -> The maintenance window feature is only available for [TiDB Dedicated clusters](/tidb-cloud/select-cluster-tier.md#tidb-dedicated). +> The maintenance window feature is only available for [TiDB Cloud Dedicated clusters](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). ## Allowed and disallowed operations during a maintenance window diff --git a/tidb-cloud/configure-security-settings.md b/tidb-cloud/configure-security-settings.md index c45b5705bbd4d..c60c48dfdb050 100644 --- a/tidb-cloud/configure-security-settings.md +++ b/tidb-cloud/configure-security-settings.md @@ -1,15 +1,15 @@ --- -title: Configure Cluster Security Settings -summary: Learn how to configure the root password and allowed IP addresses to connect to your cluster. +title: Configure Cluster Password Settings +summary: Learn how to configure the root password to connect to your cluster. --- -# Configure Cluster Security Settings +# Configure Cluster Password Settings -For TiDB Dedicated clusters, you can configure the root password and allowed IP addresses to connect to your cluster. +For TiDB Cloud Dedicated clusters, you can configure the root password and allowed IP addresses to connect to your cluster. > **Note:** > -> For TiDB Serverless clusters, this document is inapplicable and you can refer to [TLS Connection to TiDB Serverless](/tidb-cloud/secure-connections-to-serverless-clusters.md) instead. +> For TiDB Cloud Serverless clusters, this document is inapplicable and you can refer to [TLS Connection to TiDB Cloud Serverless](/tidb-cloud/secure-connections-to-serverless-clusters.md) instead. 1. In the TiDB Cloud console, navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. @@ -17,13 +17,11 @@ For TiDB Dedicated clusters, you can configure the root password and allowed IP > > If you have multiple projects, you can click in the lower-left corner and switch to another project. -2. In the row of your target cluster, click **...** and select **Security Settings**. -3. In the **Security Settings** dialog, configure the root password and allowed IP addresses. +2. In the row of your target cluster, click **...** and select **Password Settings**. +3. Set the root password to connect to your cluster, and then click **Save**. - To allow your cluster to be accessible by any IP addresses, click **Allow Access from Anywhere**. - -4. Click **Apply**. + You can click **Auto-generate Password** to generate a random password. The generated password will not show again, so save your password in a secure location. > **Tip:** > -> If you are viewing the overview page of your cluster, you can click the **...** in the upper-right corner of the page, select **Security Settings**, and configure these settings, too. +> If you are viewing the overview page of your cluster, you can click the **...** in the upper-right corner of the page, select **Password Settings**, and configure these settings, too. diff --git a/tidb-cloud/configure-sql-users.md b/tidb-cloud/configure-sql-users.md new file mode 100644 index 0000000000000..0b09ffbfc2405 --- /dev/null +++ b/tidb-cloud/configure-sql-users.md @@ -0,0 +1,121 @@ +--- +title: Manage Database Users and Roles +summary: Learn how to manage database users and roles in the TiDB Cloud console. +--- + +# Manage Database Users and Roles + +This document describes how to manage database users and roles using the **SQL Users** page in the [TiDB Cloud console](https://tidbcloud.com/). + +> **Note:** +> +> - The **SQL Users** page is in beta and is only available upon request. To request this feature, click **?** in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com) and click **Request Support**. Then, fill in "Apply for the SQL Users page" in the **Description** field and click **Submit**. +> - Database users and roles are independent of [organization and project users and roles](/tidb-cloud/manage-user-access.md). Database users are used to access databases in a TiDB cluster, while organization and project users are used to access organizations and projects in the [TiDB Cloud console](https://tidbcloud.com/). +> - In addition to the **SQL Users** page, you can also manage database users and roles by connecting to your cluster with a SQL client and writing SQL statements. For more information, see [TiDB User Account Management](https://docs.pingcap.com/tidb/dev/user-account-management). + +## Roles of database users + +In TiDB Cloud, you can grant both a built-in role and multiple custom roles (if available) to a SQL user for role-based access control. + +- Built-in roles + + TiDB Cloud provides the following built-in roles to help you control the database access of SQL users. You can grant one of the built-in roles to a SQL user. + + - `Database Admin` + - `Database Read-Write` + - `Database Read-Only` + +- Custom roles + + In addition to a built-in role, if your cluster has custom roles that are created using the [`CREATE ROLE`](/sql-statements/sql-statement-create-role.md) statement, you can also grant these custom roles to a SQL user when you create or edit SQL users in the TiDB Cloud console. + +After a SQL user is granted both a built-in role and multiple custom roles, the user's permissions will be the union of all the permissions derived from these roles. + +## Prerequisites + +- To manage database users and roles using the **SQL Users** page, you must be in the `Organization Owner` role of your organization or the `Project Owner` role of your project. +- If you are in the `Project Data Access Read-Write` or `Project Data Access Read-Only` role of a project, you can only view database users on the **SQL Users** page of that project. + +## Create a SQL user + +To create a SQL user, take the following steps: + +1. In the [TiDB Cloud console](https://tidbcloud.com/), go to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + +2. Click your cluster name, and then click **SQL Users** in the left navigation pane. + +3. Click **Create SQL User** in the upper-right corner. + + A dialog for the SQL user configuration is displayed. + +4. In the dialog, provide the information of the SQL user as follows: + + 1. Enter the name of the SQL user. + 2. Either create a password for the SQL user or let TiDB Cloud automatically generate a password for the user. + 3. Grant roles to the SQL user. + + - **Built-in Role**: you need to select a built-in role for the SQL user in the **Built-in Role** drop-down list. + + - **Custom Role**: if your cluster has custom roles that are created using the [`CREATE ROLE`](/sql-statements/sql-statement-create-role.md) statement, you can grant custom roles to the SQL user by selecting the roles from the **Custom Role** drop-down list. Otherwise, the **Custom Roles** drop-down list is invisible here. + + For each SQL user, you can grant a built-in role and multiple custom roles (if any). + +5. Click **Create**. + +## View SQL users + +To view SQL users of a cluster, take the following steps: + +1. In the [TiDB Cloud console](https://tidbcloud.com/), go to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + +2. Click your cluster name, and then click **SQL Users** in the left navigation pane. + +## Edit a SQL user + +To edit the password or roles of a SQL user, take the following steps: + +1. In the [TiDB Cloud console](https://tidbcloud.com/), go to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + +2. Click your cluster name, and then click **SQL Users** in the left navigation pane. + +3. In the row of the SQL user to be edited, click **...** in the **Action** column, and then click **Edit**. + + A dialog for the SQL user configuration is displayed. + +4. In the dialog, you can edit the user password and roles as needed, and then click **Update**. + + > **Note:** + > + > The roles of the default `.root` user do not support modification. You can only change the password. + +## Delete a SQL user + +To delete a SQL user, take the following steps: + +1. In the [TiDB Cloud console](https://tidbcloud.com/), go to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + +2. Click your cluster name, and then click **SQL Users** in the left navigation pane. + +3. In the row of the SQL user to be edited, click **...** in the **Action** column, and then click **Delete**. + + > **Note:** + > + > The default `.root` user does not support deletion. + +4. Confirm the deletion. diff --git a/tidb-cloud/connect-to-tidb-cluster-serverless.md b/tidb-cloud/connect-to-tidb-cluster-serverless.md index 774d4a69336d2..604786f4776c1 100644 --- a/tidb-cloud/connect-to-tidb-cluster-serverless.md +++ b/tidb-cloud/connect-to-tidb-cluster-serverless.md @@ -1,33 +1,61 @@ --- -title: Connect to Your TiDB Serverless Cluster -summary: Learn how to connect to your TiDB Serverless cluster via different methods. +title: Connect to Your TiDB Cloud Serverless Cluster +summary: Learn how to connect to your TiDB Cloud Serverless cluster via different methods. --- -# Connect to Your TiDB Serverless Cluster +# Connect to Your TiDB Cloud Serverless Cluster -This document introduces the methods to connect to your TiDB Serverless cluster. +This document describes how to connect to your TiDB Cloud Serverless cluster. > **Tip:** > -> To learn how to connect to a TiDB Dedicated cluster, see [Connect to Your TiDB Dedicated Cluster](/tidb-cloud/connect-to-tidb-cluster.md). +> To learn how to connect to a TiDB Cloud Dedicated cluster, see [Connect to Your TiDB Cloud Dedicated Cluster](/tidb-cloud/connect-to-tidb-cluster.md). -After your TiDB Serverless cluster is created on TiDB Cloud, you can connect to it via one of the following methods: +## Connection methods -- [Connect via private endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md) (recommended) +After your TiDB Cloud Serverless cluster is created on TiDB Cloud, you can connect to it via one of the following methods: - Private endpoint connection provides a private endpoint to allow SQL clients in your VPC to securely access services over AWS PrivateLink, which provides highly secure and one-way access to database services with simplified network management. +- Direct connections + + Direct connections mean the MySQL native connection system over TCP. You can connect to your TiDB Cloud Serverless cluster using any tool that supports MySQL connection, such as [MySQL client](https://dev.mysql.com/doc/refman/8.0/en/mysql.html). + +- [Data Service (beta)](/tidb-cloud/data-service-overview.md) + + TiDB Cloud provides a Data Service feature that enables you to connect to your TiDB Cloud Serverless cluster via an HTTPS request using a custom API endpoint. Unlike direct connections, Data Service accesses TiDB Cloud Serverless data via a RESTful API rather than raw SQL. + +- [Serverless Driver (beta)](/tidb-cloud/serverless-driver.md) + + TiDB Cloud provides a serverless driver for JavaScript, which allows you to connect to your TiDB Cloud Serverless cluster in edge environments with the same experience as direct connections. -- [Connect via public endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md) +In the preceding connection methods, you can choose your desired one based on your needs: + +| Connection method | User interface | Scenario | +|--------------------|--------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Direct connections | SQL/ORM | Long-running environment, such as Java, Node.js, and Python. | +| Data Service | RESTful API | All browser and application interactions. | +| Serverless Driver | SQL/ORM | Serverless and edge environments such as [Vercel Edge Functions](https://vercel.com/docs/functions/edge-functions) and [Cloudflare Workers](https://workers.cloudflare.com/). | + +## Network + +There are two network connection types for TiDB Cloud Serverless: + +- [Private endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md) (recommended) + + Private endpoint connection provides a private endpoint to allow SQL clients in your VPC to securely access services over AWS PrivateLink, which provides highly secure and one-way access to database services with simplified network management. - The standard connection exposes a public endpoint with traffic filters, so you can connect to your TiDB cluster via a SQL client from your laptop. +- [Public endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md) - TiDB Serverless supports [TLS connections](/tidb-cloud/secure-connections-to-serverless-clusters.md), which ensures the security of data transmission from your applications to TiDB clusters. + The standard connection exposes a public endpoint, so you can connect to your TiDB cluster via a SQL client from your laptop. -- [Connect via Chat2Query (beta)](/tidb-cloud/explore-data-with-chat2query.md) + TiDB Cloud Serverless requires [TLS connections](/tidb-cloud/secure-connections-to-serverless-clusters.md), which ensures the security of data transmission from your applications to TiDB clusters. - TiDB Cloud is powered by artificial intelligence (AI). You can use Chat2Query (beta), an AI-powered SQL editor in the [TiDB Cloud console](https://tidbcloud.com/), to maximize your data value. +The following table shows the network you can use in different connection methods: - In Chat2Query, you can either simply type `--` followed by your instructions to let AI generate SQL queries automatically or write SQL queries manually, and then run SQL queries against databases without a terminal. You can find the query results in tables intuitively and check the query logs easily. +| Connection method | Network | Description | +|----------------------------|------------------------------|-------------------------------------------------------------------------------------------------------------------| +| Direct connections | Public or private endpoint | Direct connections can be made via both public and private endpoints. | +| Data Service (beta) | / | Accessing TiDB Cloud Serverless via Data Service (beta) does not need to specify the network type. | +| Serverless Driver (beta) | Public endpoint | Serverless Driver only supports connections via public endpoint. | ## What's next diff --git a/tidb-cloud/connect-to-tidb-cluster.md b/tidb-cloud/connect-to-tidb-cluster.md index 2c36e21103fdf..7fb8a90c76df9 100644 --- a/tidb-cloud/connect-to-tidb-cluster.md +++ b/tidb-cloud/connect-to-tidb-cluster.md @@ -1,45 +1,48 @@ --- -title: Connect to Your TiDB Dedicated Cluster -summary: Learn how to connect to your TiDB Dedicated cluster via different methods. +title: Connect to Your TiDB Cloud Dedicated Cluster +summary: Learn how to connect to your TiDB Cloud Dedicated cluster via different methods. --- -# Connect to Your TiDB Dedicated Cluster +# Connect to Your TiDB Cloud Dedicated Cluster -This document introduces the methods to connect to your TiDB Dedicated cluster. +This document introduces the methods to connect to your TiDB Cloud Dedicated cluster. > **Tip:** > -> To learn how to connect to a TiDB Serverless cluster, see [Connect to Your TiDB Serverless Cluster](/tidb-cloud/connect-to-tidb-cluster-serverless.md). +> To learn how to connect to a TiDB Cloud Serverless cluster, see [Connect to Your TiDB Cloud Serverless Cluster](/tidb-cloud/connect-to-tidb-cluster-serverless.md). -After your TiDB Dedicated cluster is created on TiDB Cloud, you can connect to it via one of the following methods: +After your TiDB Cloud Dedicated cluster is created on TiDB Cloud, you can connect to it via one of the following methods: -- [Connect via standard connection](/tidb-cloud/connect-via-standard-connection.md) +- Direct connections - The standard connection exposes a public endpoint with traffic filters, so you can connect to your TiDB cluster via a SQL client from your laptop. You can connect to your TiDB clusters using TLS, which ensures the security of data transmission from your applications to TiDB clusters. + Direct connections use the MySQL native connection system over TCP. You can connect to your TiDB Cloud Dedicated cluster using any tool that supports MySQL connections, such as the [MySQL Command-Line Client](https://dev.mysql.com/doc/refman/8.0/en/mysql.html). TiDB Cloud also provides [SQL Shell](/tidb-cloud/connect-via-sql-shell.md), which enables you to try TiDB SQL, test out TiDB's compatibility with MySQL quickly, and manage user privileges. -- [Connect via private endpoint with AWS](/tidb-cloud/set-up-private-endpoint-connections.md) (recommended) + TiDB Cloud Dedicated provides three network connection types: - For TiDB Dedicated clusters hosted on AWS, private endpoint connection provides a private endpoint to allow SQL clients in your VPC to securely access services over AWS PrivateLink, which provides highly secure and one-way access to database services with simplified network management. + - [Public connection](/tidb-cloud/connect-via-standard-connection.md) -- [Connect via private endpoint with Google Cloud](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md) (recommended) + The public connection exposes a public endpoint with traffic filters, so you can connect to your TiDB cluster via a SQL client from your laptop. You can connect to your TiDB clusters using TLS, which ensures the security of data transmission from your applications to TiDB clusters. For more information, see [Connect to TiDB Cloud Dedicated via Public Connection](/tidb-cloud/connect-via-standard-connection.md). - For TiDB Dedicated clusters hosted on Google Cloud, private endpoint connection provides a private endpoint to allow SQL clients in your VPC to securely access services over Google Cloud Private Service Connect, which provides highly secure and one-way access to database services with simplified network management. + - Private endpoint (recommended) -- [Connect via VPC peering](/tidb-cloud/set-up-vpc-peering-connections.md) + Private endpoint connection provides a private endpoint to allow SQL clients in your VPC to securely access TiDB Cloud Dedicated clusters. This uses the private link service provided by different cloud providers, which provides highly secure and one-way access to database services with simplified network management. - If you want lower latency and more security, set up VPC peering and connect via a private endpoint using a VM instance on the corresponding cloud provider in your cloud account. + - For TiDB Cloud Dedicated clusters hosted on AWS, the private endpoint connection uses AWS PrivateLink. For more information, see [Connect to a TiDB Cloud Dedicated Cluster via Private Endpoint with AWS](/tidb-cloud/set-up-private-endpoint-connections.md). + - For TiDB Cloud Dedicated clusters hosted on Google Cloud, the private endpoint connection uses Google Cloud Private Service Connect. For more information, see [Connect to a TiDB Cloud Dedicated Cluster via Google Cloud Private Service Connect](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md). -- [Connect via Chat2Query (beta)](/tidb-cloud/explore-data-with-chat2query.md) + - [VPC peering](/tidb-cloud/set-up-vpc-peering-connections.md) + + If you want lower latency and more security, set up VPC peering and connect via a private endpoint using a VM instance on the corresponding cloud provider in your cloud account. For more information, see [Connect to TiDB Cloud Dedicated via VPC Peering](/tidb-cloud/set-up-vpc-peering-connections.md). + +- [Built-in SQL Editor](/tidb-cloud/explore-data-with-chat2query.md) > **Note:** > - > To use Chat2Query on [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters, contact [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md). - - TiDB Cloud is powered by artificial intelligence (AI). If your cluster is hosted on AWS and the TiDB version of the cluster is v6.5.0 or later, you can use Chat2Query (beta), an AI-powered SQL editor in the [TiDB Cloud console](https://tidbcloud.com/), to maximize your data value. + > To use SQL Editor on [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters, contact [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md). - In Chat2Query, you can either simply type `--` followed by your instructions to let AI generate SQL queries automatically or write SQL queries manually, and then run SQL queries against databases without a terminal. You can find the query results in tables intuitively and check the query logs easily. + If your cluster is hosted on AWS and the TiDB version of the cluster is v6.5.0 or later, you can use the AI-assisted SQL Editor in the [TiDB Cloud console](https://tidbcloud.com/) to maximize your data value. -- [Connect via SQL Shell](/tidb-cloud/connect-via-sql-shell.md): to try TiDB SQL and test out TiDB's compatibility with MySQL quickly, or administer user privileges. + In SQL Editor, you can either write SQL queries manually or simply press + I on macOS (or Control + I on Windows or Linux) to instruct [Chat2Query (beta)](/tidb-cloud/tidb-cloud-glossary.md#chat2query) to generate SQL queries automatically. This enables you to run SQL queries against databases without a local SQL client. You can intuitively view the query results in tables or charts and easily check the query logs. ## What's next diff --git a/tidb-cloud/connect-via-sql-shell.md b/tidb-cloud/connect-via-sql-shell.md index a6d3c26e1b6fe..5fa72f648bb95 100644 --- a/tidb-cloud/connect-via-sql-shell.md +++ b/tidb-cloud/connect-via-sql-shell.md @@ -9,7 +9,7 @@ In TiDB Cloud SQL Shell, you can try TiDB SQL, test out TiDB's compatibility wit > **Note:** > -> You cannot connect to [TiDB Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-serverless) using SQL Shell. To connect to your TiDB Serverless cluster, see [Connect to TiDB Serverless clusters](/tidb-cloud/connect-to-tidb-cluster-serverless.md). +> You cannot connect to [TiDB Cloud Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) using SQL Shell. To connect to your TiDB Cloud Serverless cluster, see [Connect to TiDB Cloud Serverless clusters](/tidb-cloud/connect-to-tidb-cluster-serverless.md). To connect to your TiDB cluster using SQL shell, perform the following steps: @@ -19,8 +19,6 @@ To connect to your TiDB cluster using SQL shell, perform the following steps: > > If you have multiple projects, you can click in the lower-left corner and switch to another project. -2. Click the name of your target cluster to go to its cluster overview page, and then click **Connect** in the upper-right corner. A connection dialog is displayed. - -3. In the dialog, select the **Web SQL Shell** tab, and then click **Open SQL Shell**. - +2. Click the name of your target cluster to go to its cluster overview page, and then click **Networking** in the left navigation pane. +3. On the **Networking** page, click **Web SQL Shell** in the upper-right corner. 4. On the prompted **Enter password** line, enter the root password of the current cluster. Then your application is connected to the TiDB cluster. \ No newline at end of file diff --git a/tidb-cloud/connect-via-standard-connection-serverless.md b/tidb-cloud/connect-via-standard-connection-serverless.md index 39e06b4aab3c9..04b7d748ee56e 100644 --- a/tidb-cloud/connect-via-standard-connection-serverless.md +++ b/tidb-cloud/connect-via-standard-connection-serverless.md @@ -1,36 +1,62 @@ --- -title: Connect to TiDB Serverless via Public Endpoint -summary: Learn how to connect to your TiDB Serverless cluster via public endpoint. +title: Connect to TiDB Cloud Serverless via Public Endpoint +summary: Learn how to connect to your TiDB Cloud Serverless cluster via public endpoint. --- -# Connect to TiDB Serverless via Public Endpoint +# Connect to TiDB Cloud Serverless via Public Endpoint -This document describes how to connect to your TiDB Serverless cluster via public endpoint. With the public endpoint, you can connect to your TiDB Serverless cluster via a SQL client from your laptop. +This document describes how to connect to your TiDB Cloud Serverless cluster via a public endpoint, using a SQL client from your computer, as well as how to disable a public endpoint. + +## Connect via a public endpoint > **Tip:** > -> To learn how to connect to a TiDB Dedicated cluster via public endpoint, see [Connect to TiDB Dedicated via Standard Connection](/tidb-cloud/connect-via-standard-connection.md). +> To learn how to connect to a TiDB Cloud Dedicated cluster via public endpoint, see [Connect to TiDB Cloud Dedicated via Public Connection](/tidb-cloud/connect-via-standard-connection.md). -To connect to a TiDB Serverless cluster via public endpoint, take the following steps: +To connect to a TiDB Cloud Serverless cluster via public endpoint, take the following steps: 1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. In the dialog, keep the default setting of the endpoint type as `Public`, and select your preferred connection method and operating system to get the corresponding connection string. +3. In the dialog, keep the default setting of the connection type as `Public`, and select your preferred connection method and operating system to get the corresponding connection string. > **Note:** > - > - Keeping the endpoint type as `Public` means the connection is via standard TLS connection. For more information, see [TLS Connection to TiDB Serverless](/tidb-cloud/secure-connections-to-serverless-clusters.md). - > - If you choose **Private** in the **Endpoint Type** drop-down list, it means that the connection is via private endpoint. For more information, see [Connect to TiDB Serverless via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md). + > - Keeping the connection type as `Public` means the connection is via standard TLS connection. For more information, see [TLS Connection to TiDB Cloud Serverless](/tidb-cloud/secure-connections-to-serverless-clusters.md). + > - If you choose **Private Endpoint** in the **Connection Type** drop-down list, it means that the connection is via private endpoint. For more information, see [Connect to TiDB Cloud Serverless via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md). + +4. TiDB Cloud Serverless lets you create [branches](/tidb-cloud/branch-overview.md) for your cluster. After a branch is created, you can choose to connect to the branch via the **Branch** drop-down list. `main` represents the cluster itself. -4. If you have not set a password yet, click **Create password** to generate a random password. The generated password will not show again, so save your password in a secure location. +5. If you have not set a password yet, click **Generate Password** to generate a random password. The generated password will not show again, so save your password in a secure location. -5. Connect to your cluster with the connection string. +6. Connect to your cluster with the connection string. > **Note:** > - > When you connect to a TiDB Serverless cluster, you must include the prefix for your cluster in the user name and wrap the name with quotation marks. For more information, see [User name prefix](/tidb-cloud/select-cluster-tier.md#user-name-prefix). + > When you connect to a TiDB Cloud Serverless cluster, you must include the prefix for your cluster in the user name and wrap the name with quotation marks. For more information, see [User name prefix](/tidb-cloud/select-cluster-tier.md#user-name-prefix). + +## Disable a public endpoint + +If you do not need to use a public endpoint of a TiDB Cloud Serverless cluster, you can disable it to prevent connections from the internet: + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Networking** in the left navigation pane and click **Disable** in the right pane. A confirmation dialog is displayed. + +3. Click **Disable** in the confirmation dialog. + +After disabling the public endpoint, the `Public` entry in the **Connection Type** drop-down list of the connect dialog is disabled. If users are still trying to access the cluster from the public endpoint, they will get an error. + +> **Note:** +> +> Disabling the public endpoint does not affect existing connections. It only prevents new connections from the internet. + +You can re-enable the public endpoint after disabling it: + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Networking** in the left navigation pane and click **Enable** in the right pane. ## What's next diff --git a/tidb-cloud/connect-via-standard-connection.md b/tidb-cloud/connect-via-standard-connection.md index 922b94825cffc..9b7cecd457c6b 100644 --- a/tidb-cloud/connect-via-standard-connection.md +++ b/tidb-cloud/connect-via-standard-connection.md @@ -1,17 +1,23 @@ --- -title: Connect to TiDB Dedicated via Standard Connection -summary: Learn how to connect to your TiDB Cloud cluster via standard connection. +title: Connect to TiDB Cloud Dedicated via Public Connection +summary: Learn how to connect to your TiDB Cloud cluster via public connection. --- -# Connect to TiDB Dedicated via Standard Connection +# Connect to TiDB Cloud Dedicated via Public Connection -This document describes how to connect to your TiDB Dedicated cluster via standard connection. The standard connection exposes a public endpoint with traffic filters, so you can connect to your TiDB Dedicated cluster via a SQL client from your laptop. +This document describes how to connect to your TiDB Cloud Dedicated cluster via public connection. The public connection exposes a public endpoint with traffic filters, so you can connect to your TiDB Cloud Dedicated cluster via a SQL client from your laptop. > **Tip:** > -> To learn how to connect to a TiDB Serverless cluster via standard connection, see [Connect to TiDB Serverless via Public Endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md). +> To learn how to connect to a TiDB Cloud Serverless cluster via public connection, see [Connect to TiDB Cloud Serverless via Public Endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md). -To connect to a TiDB Dedicated cluster via standard connection, take the following steps: +## Prerequisite: Configure IP access list + +For public connections, TiDB Cloud Dedicated only allows client connections from addresses in the IP access list. If you have not configured the IP access list, follow the steps in [Configure an IP Access List](/tidb-cloud/configure-ip-access-list.md) to configure it before your first connection. + +## Connect to the cluster + +To connect to a TiDB Cloud Dedicated cluster via public connection, take the following steps: 1. Open the overview page of the target cluster. @@ -25,29 +31,13 @@ To connect to a TiDB Dedicated cluster via standard connection, take the followi 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Create a traffic filter for the cluster. Traffic filter is a list of IPs and CIDR addresses that are allowed to access TiDB Cloud via a SQL client. - - If the traffic filter is already set, skip the following sub-steps. If the traffic filter is empty, take the following sub-steps to add one. - - 1. Click one of the buttons to add some rules quickly. - - - **Add My Current IP Address** - - **Allow Access from Anywhere** - - 2. Provide an optional description for the newly added IP address or CIDR range. - - 3. Click **Create Filter** to confirm the changes. - -4. Under **Step 2: Download TiDB cluster CA** in the dialog, click **Download TiDB cluster CA** for TLS connection to TiDB clusters. The TiDB cluster CA supports TLS 1.2 version by default. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list. - > **Note:** - > - > - The TiDB cluster CA is only available for TiDB Dedicated clusters. - > - Currently, TiDB Cloud only provides the connection strings and sample code for these connection methods: MySQL, MyCLI, JDBC, Python, Go, and Node.js. + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](/tidb-cloud/configure-ip-access-list.md) to configure it before your first connection. -5. Under **Step 3: Connect with a SQL client** in the dialog, click the tab of your preferred connection method, and then refer to the connection string and sample code on the tab to connect to your cluster. +4. Click **CA cert** to download CA cert for TLS connection to TiDB clusters. The CA cert supports TLS 1.2 version by default. - Note that you need to use the path of the downloaded CA file as the argument of the `--ssl-ca` option in the connection string. +5. Choose your preferred connection method, and then refer to the connection string and sample code on the tab to connect to your cluster. ## What's next diff --git a/tidb-cloud/cppo-customer.md b/tidb-cloud/cppo-customer.md new file mode 100644 index 0000000000000..80a4ebdfa417e --- /dev/null +++ b/tidb-cloud/cppo-customer.md @@ -0,0 +1,32 @@ +--- +title: Reseller's Customer +summary: Learn how to become a reseller's customer. +--- + +# Reseller's Customer + +A reseller's customer subscribes to TiDB Cloud services through a reseller in the AWS Marketplace. + +Unlike direct TiDB Cloud customers, a reseller's customers pay invoices via the AWS Marketplace rather than directly to PingCAP. However, all other daily operations in the TiDB Cloud console remain the same for both AWS Marketplace Channel Partner Private Offer (CPPO) and direct TiDB Cloud customers. + +This document outlines how to check both future and historical bills. + +## Migrate from a direct TiDB Cloud account to a reseller's customer account + +> **Tip:** +> +> Direct customers are the end customers who directly purchase TiDB Cloud from and pay invoices to PingCAP. + +If you are currently a direct customer with a TiDB Cloud account, you can ask your reseller to migrate your account to a reseller's customer account. + +The migration will take effect on the first day of a future month. Discuss with your reseller to determine the specific effective date. + +On the effective day of migration, you will receive an email notification. + +## View your future bills + +As a reseller's customer, you will have your invoices processed and paid through the AWS Marketplace. + +## View your historical bills + +If you have migrated from a direct TiDB Cloud account to a reseller's customer account, you can still access your billing history before the migration. To do so, go to **Billing** > **Bills** > **History** in the TiDB Cloud console. diff --git a/tidb-cloud/create-tidb-cluster-serverless.md b/tidb-cloud/create-tidb-cluster-serverless.md index 474ae6408df88..4bab3ee13498d 100644 --- a/tidb-cloud/create-tidb-cluster-serverless.md +++ b/tidb-cloud/create-tidb-cluster-serverless.md @@ -1,15 +1,15 @@ --- -title: Create a TiDB Serverless Cluster -summary: Learn how to create your TiDB Serverless cluster. +title: Create a TiDB Cloud Serverless Cluster +summary: Learn how to create your TiDB Cloud Serverless cluster. --- -# Create a TiDB Serverless Cluster +# Create a TiDB Cloud Serverless Cluster -This document describes how to create a TiDB Serverless cluster in the [TiDB Cloud console](https://tidbcloud.com/). +This document describes how to create a TiDB Cloud Serverless cluster in the [TiDB Cloud console](https://tidbcloud.com/). > **Tip:** > -> To learn how to create a TiDB Dedicated cluster, see [Create a TiDB Dedicated Cluster](/tidb-cloud/create-tidb-cluster.md). +> To learn how to create a TiDB Cloud Dedicated cluster, see [Create a TiDB Cloud Dedicated Cluster](/tidb-cloud/create-tidb-cluster.md). ## Before you begin @@ -21,7 +21,7 @@ If you do not have a TiDB Cloud account, click [here](https://tidbcloud.com/sign ## Steps -If you are in the `Organization Owner` or the `Project Owner` role, you can create a TiDB Serverless cluster as follows: +If you are in the `Organization Owner` or the `Project Owner` role, you can create a TiDB Cloud Serverless cluster as follows: 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/), and then navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page. @@ -29,22 +29,24 @@ If you are in the `Organization Owner` or the `Project Owner` role, you can crea 3. On the **Create Cluster** page, **Serverless** is selected by default. -4. The cloud provider of TiDB Serverless is AWS. You can select an AWS region where you want to host your cluster. +4. The cloud provider of TiDB Cloud Serverless is AWS. You can select an AWS region where you want to host your cluster. -5. (Optional) Change the spending limit if you plan to use more storage and compute resources than the [free quota](/tidb-cloud/select-cluster-tier.md#usage-quota). If you have not added a payment method, you need to add a credit card after editing the limit. +5. Update the default cluster name if necessary. + +6. Select a cluster plan. TiDB Cloud Serverless provides two [cluster plans](/tidb-cloud/select-cluster-tier.md#cluster-plans): **Free Cluster** and **Scalable Cluster**. You can start with a free cluster and later upgrade to a scalable cluster as your needs grow. To create a scalable cluster, you need to specify a **Monthly Spending Limit** and add a credit card. > **Note:** > - > For each organization in TiDB Cloud, you can create a maximum of five TiDB Serverless clusters by default. To create more TiDB Serverless clusters, you need to add a credit card and set a [spending limit](/tidb-cloud/tidb-cloud-glossary.md#spending-limit) for the usage. + > For each organization in TiDB Cloud, you can create a maximum of five [free clusters](/tidb-cloud/select-cluster-tier.md#free-cluster-plan) by default. To create more TiDB Cloud Serverless clusters, you need to add a credit card and create [scalable clusters](/tidb-cloud/select-cluster-tier.md#scalable-cluster-plan) for the usage. -6. Update the default cluster name if necessary, and then click **Create**. +7. Click **Create**. The cluster creation process starts and your TiDB Cloud cluster will be created in approximately 30 seconds. ## What's next -After your cluster is created, follow the instructions in [Connect to TiDB Serverless via Public Endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md) to create a password for your cluster. +After your cluster is created, follow the instructions in [Connect to TiDB Cloud Serverless via Public Endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md) to create a password for your cluster. > **Note:** > -> If you do not set a password, you cannot connect to the cluster. \ No newline at end of file +> If you do not set a password, you cannot connect to the cluster. diff --git a/tidb-cloud/create-tidb-cluster.md b/tidb-cloud/create-tidb-cluster.md index 520ba710d4509..274de0ba9bae1 100644 --- a/tidb-cloud/create-tidb-cluster.md +++ b/tidb-cloud/create-tidb-cluster.md @@ -1,15 +1,15 @@ --- -title: Create a TiDB Dedicated Cluster -summary: Learn how to create your TiDB Dedicated cluster. +title: Create a TiDB Cloud Dedicated Cluster +summary: Learn how to create your TiDB Cloud Dedicated cluster. --- -# Create a TiDB Dedicated Cluster +# Create a TiDB Cloud Dedicated Cluster -This tutorial guides you through signing up and creating a TiDB Dedicated cluster. +This tutorial guides you through signing up and creating a TiDB Cloud Dedicated cluster. > **Tip:** > -> To learn how to create a TiDB Serverless cluster, see [Create a TiDB Serverless Cluster](/tidb-cloud/create-tidb-cluster-serverless.md). +> To learn how to create a TiDB Cloud Serverless cluster, see [Create a TiDB Cloud Serverless Cluster](/tidb-cloud/create-tidb-cluster-serverless.md). ## Before you begin @@ -27,9 +27,7 @@ If you are an organization owner, you can rename the default project or create a 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/), and then click in the lower-left corner. -2. Click **Organization Settings**. - - The **Projects** tab is displayed by default. +2. Click **Organization Settings**, and click the **Projects** tab in the left navigation pane. The **Projects** tab is displayed. 3. Do one of the following: @@ -38,9 +36,9 @@ If you are an organization owner, you can rename the default project or create a 4. To return to the cluster page, click the TiDB Cloud logo in the upper-left corner of the window. -## Step 2. Create a TiDB Dedicated cluster +## Step 2. Create a TiDB Cloud Dedicated cluster -If you are in the `Organization Owner` or the `Project Owner` role, you can create a TiDB Dedicated cluster as follows: +If you are in the `Organization Owner` or the `Project Owner` role, you can create a TiDB Cloud Dedicated cluster as follows: 1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. @@ -61,11 +59,12 @@ If you are in the `Organization Owner` or the `Project Owner` role, you can crea 2. Configure the [cluster size](/tidb-cloud/size-your-cluster.md) for TiDB, TiKV, and TiFlash (optional) respectively. 3. Update the default cluster name and port number if necessary. - 4. If this is the first cluster of your current project and CIDR has not been configured for this project, you need to set the project CIDR. If you do not see the **Project CIDR** field, it means that CIDR has already been configured for this project. + 4. If CIDR has not been configured for this region, you need to set the CIDR. If you do not see the **Project CIDR** field, it means that CIDR has already been configured for this region. > **Note:** > - > When setting the project CIDR, avoid any conflicts with the CIDR of the VPC where your application is located. You cannot modify your project CIDR once it is set. + > - TiDB Cloud will create a VPC with this CIDR when the first cluster in this region is created. All the subsequent clusters of the same project in this region will use this VPC. + > - When setting the CIDR, avoid any conflicts with the CIDR of the VPC where your application is located. You cannot modify your CIDR once the VPC is created. 4. Confirm the cluster and billing information on the right side. @@ -79,14 +78,16 @@ If you are in the `Organization Owner` or the `Project Owner` role, you can crea Your TiDB Cloud cluster will be created in approximately 20 to 30 minutes. -## Step 3. Configure secure settings +## Step 3. Set the root password + +After your cluster is created, take the following steps to set the root password: -After your cluster is created, take the following steps to configure the security settings: +1. In the upper-right corner of your cluster overview page, click **...** and select **Password Settings**. -1. In the upper-right corner of your cluster overview page, click **...** and select **Security Settings**. +2. Set the root password to connect to your cluster, and then click **Save**. -2. Set the root password and allowed IP addresses to connect to your cluster, and then click **Apply**. + You can click **Auto-generate Password** to generate a random password. The generated password will not show again, so save your password in a secure location. ## What's next -After your cluster is created on TiDB Cloud, you can connect to it via the methods provided in [Connect to Your TiDB Dedicated Cluster](/tidb-cloud/connect-via-standard-connection-serverless.md). \ No newline at end of file +After your cluster is created on TiDB Cloud, you can connect to it via the methods provided in [Connect to Your TiDB Cloud Dedicated Cluster](/tidb-cloud/connect-to-tidb-cluster.md). diff --git a/tidb-cloud/csv-config-for-import-data.md b/tidb-cloud/csv-config-for-import-data.md index 2e0419a47fc04..fa6c7f556937c 100644 --- a/tidb-cloud/csv-config-for-import-data.md +++ b/tidb-cloud/csv-config-for-import-data.md @@ -33,12 +33,6 @@ The following is the CSV Configuration window when you use the Import Data servi - Default: `"` -## With header - -- Definition: whether *all* CSV files contain a header row. If **With header** is `True`, the first row is used as the column names. If **With header** is `False`, the first row is treated as an ordinary data row. - -- Default: `True` - ## Backslash escape - Definition: whether to parse backslash inside fields as escape characters. If **Backslash escape** is `True`, the following sequences are recognized and converted: diff --git a/tidb-cloud/data-service-api-key.md b/tidb-cloud/data-service-api-key.md index b1e8d7907d1db..368ffd66075a8 100644 --- a/tidb-cloud/data-service-api-key.md +++ b/tidb-cloud/data-service-api-key.md @@ -22,16 +22,64 @@ The TiDB Cloud Data API supports both [Basic Authentication](https://en.wikipedi ## Rate limiting -Request quotas are rate-limited as follows: +Request quotas are subject to rate limits as follows: -- 100 requests per minute (rpm) per API key -- 100 requests per day for each Chat2Query Data App +- TiDB Cloud Data Service allows up to 100 requests per minute (rpm) per API key by default. -If you exceed the rate limit, the API returns a `429` error. To increase your quota, you can [submit a request](https://support.pingcap.com/hc/en-us/requests/new?ticket_form_id=7800003722519) to our support team. + You can edit the rate limit of an API key when you [create](#create-an-api-key) or [edit](#edit-an-api-key) the key. The supported value range is from `1` to `1000`. If your requests per minute exceed the rate limit, the API returns a `429` error. To get a quota of more than 1000 rpm per API key, you can [submit a request](https://support.pingcap.com/hc/en-us/requests/new?ticket_form_id=7800003722519) to our support team. + + Each API request returns the following headers about the limit. + + - `X-Ratelimit-Limit-Minute`: The number of requests allowed per minute. + - `X-Ratelimit-Remaining-Minute`: The number of remaining requests in the current minute. When it reaches `0`, the API returns a `429` error and indicates that you exceed the rate limit. + - `X-Ratelimit-Reset`: The time in seconds at which the current rate limit resets. + + If you exceed the rate limit, an error response returns like this: + + ```bash + HTTP/2 429 + date: Mon, 05 Sep 2023 02:50:52 GMT + content-type: application/json + content-length: 420 + x-debug-trace-id: 202309040250529dcdf2055e7b2ae5e9 + x-ratelimit-reset: 8 + x-ratelimit-remaining-minute: 0 + x-ratelimit-limit-minute: 10 + x-kong-response-latency: 1 + server: kong/2.8.1 + + {"type":"","data":{"columns":[],"rows":[],"result":{"latency":"","row_affect":0,"code":49900007,"row_count":0,"end_ms":0,"limit":0,"message":"API key rate limit exceeded. The limit can be increased up to 1000 requests per minute per API key in TiDB Cloud console. For an increase in quota beyond 1000 rpm, please contact us: https://support.pingcap.com/hc/en-us/requests/new?ticket_form_id=7800003722519","start_ms":0}}} + ``` + +- TiDB Cloud Data Service allows up to 100 requests per day for each Chat2Query Data App. + +## API key expiration + +By default, API keys never expire. However, for security considerations, you can specify an expiration time for your API key when you [create](#create-an-api-key) or [edit](#edit-an-api-key) the key. + +- An API key is valid only before its expiration time. Once expired, all requests using that key will fail with a `401` error, and the response is similar as follows: + + ```bash + HTTP/2 401 + date: Mon, 05 Sep 2023 02:50:52 GMT + content-type: application/json + content-length: 420 + x-debug-trace-id: 202309040250529dcdf2055e7b2ae5e9 + x-kong-response-latency: 1 + server: kong/2.8.1 + + {"data":{"result":{"start_ms":0,"end_ms":0,"latency":"","row_affect":0,"limit":0,"code":49900002,"message":"API Key is no longer valid","row_count":0},"columns":[],"rows":[]},"type":""} + ``` + +- You can also expire API keys manually. For detailed steps, see [Expire an API key](#expire-an-api-key) and [Expire all API keys](#expire-all-api-keys). Once you manually expire an API key, the expiration takes effect immediately. + +- You can check the status and expiration time of your API keys in the **Authentication** area of your target Data App. + +- Once expired, an API key cannot be activated or edited again. ## Manage API keys -The following sections describe how to create, edit, and delete an API key for a Data App. +The following sections describe how to create, edit, delete, and expire API keys for a Data App. ### Create an API key @@ -40,12 +88,23 @@ To create an API key for a Data App, perform the following steps: 1. Navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project. 2. In the left pane, click the name of your target Data App to view its details. 3. In the **Authentication** area, click **Create API Key**. -4. In the **Create API Key** dialog box, enter a description and select a role for your API key. +4. In the **Create API Key** dialog box, do the following: + + 1. (Optional) Enter a description for your API key. + 2. Select a role for your API key. + + The role is used to control whether the API key can read or write data to the clusters linked to the Data App. You can select the `ReadOnly` or `ReadAndWrite` role: + + - `ReadOnly`: only allows the API key to read data, such as `SELECT`, `SHOW`, `USE`, `DESC`, and `EXPLAIN` statements. + - `ReadAndWrite`: allows the API key to read and write data. You can use this API key to execute all SQL statements, such as DML and DDL statements. - The role is used to control whether the API key can read or write data to the clusters linked to the Data App. You can select the `ReadOnly` or `ReadAndWrite` role: + 3. (Optional) Set a desired rate limit for your API key. - - `ReadOnly`: only allows the API key to read data, such as `SELECT`, `SHOW`, `USE`, `DESC`, and `EXPLAIN` statements. - - `ReadAndWrite`: allows the API key to read and write data. You can use this API key to execute all SQL statements, such as DML and DDL statements. + If your requests per minute exceed the rate limit, the API returns a `429` error. To get a quota of more than 1000 requests per minute (rpm) per API key, you can [submit a request](https://support.pingcap.com/hc/en-us/requests/new?ticket_form_id=7800003722519) to our support team. + + 4. (Optional) Set a desired expiration time for your API key. + + By default, an API key never expires. If you prefer to specify an expiration time for the API key, click **Expires in**, select a time unit (`Minutes`, `Days`, or `Months`), and then fill in a desired number for the time unit. 5. Click **Next**. The public key and private key are displayed. @@ -55,12 +114,16 @@ To create an API key for a Data App, perform the following steps: ### Edit an API key -To edit the description of an API key, perform the following steps: +> **Note**: +> +> You cannot edit an expired key. + +To edit the description or rate limit of an API key, perform the following steps: 1. Navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project. 2. In the left pane, click the name of your target Data App to view its details. -3. In the **API Key** area, locate the **Action** column, and then click **...** > **Edit** in the API key row that you want to change. -4. Update the description or the role of the API key. +3. In the **Authentication** area, locate the **Action** column, and then click **...** > **Edit** in the API key row that you want to change. +4. Update the description, role, rate limit, or expiration time of the API key. 5. Click **Update**. ### Delete an API key @@ -75,3 +138,25 @@ To delete an API key for a Data App, perform the following steps: 2. In the left pane, click the name of your target Data App to view its details. 3. In the **API Key** area, locate the **Action** column, and then click **...** > **Delete** in the API key row that you want to delete. 4. In the displayed dialog box, confirm the deletion. + +### Expire an API key + +> **Note**: +> +> You cannot expire an expired key. + +To expire an API key for a Data App, perform the following steps: + +1. Navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project. +2. In the left pane, click the name of your target Data App to view its details. +3. In the **Authentication** area, locate the **Action** column, and then click **...** > **Expire Now** in the API key row that you want to expire. +4. In the displayed dialog box, confirm the expiration. + +### Expire all API keys + +To expire all API keys for a Data App, perform the following steps: + +1. Navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project. +2. In the left pane, click the name of your target Data App to view its details. +3. In the **Authentication** area, click **Expire All**. +4. In the displayed dialog box, confirm the expiration. \ No newline at end of file diff --git a/tidb-cloud/data-service-app-config-files.md b/tidb-cloud/data-service-app-config-files.md index bcd357d230ab2..ff45d0a5a5997 100644 --- a/tidb-cloud/data-service-app-config-files.md +++ b/tidb-cloud/data-service-app-config-files.md @@ -142,7 +142,8 @@ The following is an example configuration of `config.json`. In this example, the "type": "", "required": <0 | 1>, "default": "", - "description": "" + "description": "", + "is_path_parameter": } ], "settings": { @@ -172,19 +173,21 @@ The description of each field is as follows: | `cluster_id` | String | The ID of the TiDB cluster for your endpoint. You can get it from the URL of your TiDB cluster. For example, if your cluster URL is `https://tidbcloud.com/console/clusters/1234567891234567890/overview`, the cluster ID is `1234567891234567890`. | | `params` | Array | The parameters used in the endpoint. By defining parameters, you can dynamically replace the parameter value in your queries through the endpoint. In `params`, you can define one or multiple parameters. For each parameter, you need to define its `name`, `type`, `required`, and `default` fields. If your endpoint does not need any parameters. You can leave `params` empty such as `"params": []`. | | `params.name` | String | The name of the parameter. The name can only include letters, digits, and underscores (`_`) and must start with a letter or an underscore (`_`). **DO NOT** use `page` and `page_size` as parameter names, which are reserved for pagination of request results. | -| `params.type` | String | The data type of the parameter. Supported values are `string`, `number`, `integer`, and `boolean`. When using a `string` type parameter, you do not need to add quotation marks (`'` or `"`). For example, `foo` is valid for the `string` type and is processed as `"foo"`, whereas `"foo"` is processed as `"\"foo\""`. | +| `params.type` | String | The data type of the parameter. Supported values are `string`, `number`, `integer`, `boolean`, and `array`. When using a `string` type parameter, you do not need to add quotation marks (`'` or `"`). For example, `foo` is valid for the `string` type and is processed as `"foo"`, whereas `"foo"` is processed as `"\"foo\""`. | | `params.required` | Integer | Specifies whether the parameter is required in the request. Supported values are `0` (not required) and `1` (required). The default value is `0`. | -| `params.default` | String | The default value of the parameter. Make sure that the value matches the type of parameter you specified. Otherwise, the endpoint returns an error. | +| `params.enum` | String | (Optional) Specifies the value options of the parameter. This field is only valid when `params.type` is set to `string`, `number`, or `integer`. To specify multiple values, you can separate them with a comma (`,`). | +| `params.default` | String | The default value of the parameter. Make sure that the value matches the type of parameter you specified. Otherwise, the endpoint returns an error. The default value of an `ARRAY` type parameter is a string and you can use a comma (`,`) to separate multiple values. | | `params.description` | String | The description of the parameter. | -| `settings.timeout` | Integer | The timeout for the endpoint in milliseconds, which is `30000` by default. You can set it to an integer from `1` to `30000`. | +| `params.is_path_parameter` | Boolean | Specifies whether the parameter is a path parameter. If it is set to `true`, ensure that the `endpoint` field contains the corresponding parameter placeholders; otherwise, it will cause deployment failures. Conversely, if the `endpoint` field contains the corresponding parameter placeholders but this field is set to `false`, it will also cause deployment failures. | +| `settings.timeout` | Integer | The timeout for the endpoint in milliseconds, which is `30000` by default. You can set it to an integer from `1` to `60000`. | | `settings.row_limit` | Integer | The maximum number of rows that the endpoint can operate or return, which is `1000` by default. When `batch_operation` is set to `0`, you can set it to an integer from `1` to `2000`. When `batch_operation` is set to `1`, you can set it to an integer from `1` to `100`. | | `settings.enable_pagination` | Integer | Controls whether to enable the pagination for the results returned by the request. Supported values are `0` (disabled) and `1` (enabled). The default value is `0`. | | `settings.cache_enabled` | Integer | Controls whether to cache the response returned by your `GET` requests within a specified time-to-live (TTL) period. Supported values are `0` (disabled) and `1` (enabled). The default value is `0`. | | `settings.cache_ttl` | Integer | The time-to-live (TTL) period in seconds for cached response when `settings.cache_enabled` is set to `1`. You can set it to an integer from 30 to 600. During the TTL period, if you make the same `GET` requests again, Data Service returns the cached response directly instead of fetching data from the target database again, which improves your query performance. | | `tag` | String | The tag for the endpoint. The default value is `"Default"`. | -| `batch_operation` | Integer | Controls whether to enable the endpoint to operate in batch mode. Supported values are `0` (disabled) and `1` (enabled). When it is set to `1`, you can operate on multiple rows in a single request. To enable this option, make sure that the request method is `POST`, `PUT`, or `DELETE`. | +| `batch_operation` | Integer | Controls whether to enable the endpoint to operate in batch mode. Supported values are `0` (disabled) and `1` (enabled). When it is set to `1`, you can operate on multiple rows in a single request. To enable this option, make sure that the request method is `POST` or `PUT`. | | `sql_file` | String | The SQL file directory for the endpoint. For example, `"sql/GET-v1.sql"`. | -| `type` | String | The type of the endpoint, which can only be `"sql_endpoint"`. | +| `type` | String | The type of the endpoint. The value is `"system-data"` for predefined system endpoints and `"sql_endpoint"` for other endpoints. | | `return_type` | String | The response format of the endpoint, which can only be `"json"`. | ### SQL file configuration @@ -223,4 +226,4 @@ When writing a SQL file, pay attention to the following: > > - The parameter name is case-sensitive. > - The parameter cannot be a table name or column name. - > - The parameter name in the SQL file match the parameter name configured in [`http_endpoints/config.json`](#endpoint-configuration). \ No newline at end of file + > - The parameter name in the SQL file match the parameter name configured in [`http_endpoints/config.json`](#endpoint-configuration). diff --git a/tidb-cloud/data-service-custom-domain.md b/tidb-cloud/data-service-custom-domain.md new file mode 100644 index 0000000000000..dc60c8030f938 --- /dev/null +++ b/tidb-cloud/data-service-custom-domain.md @@ -0,0 +1,72 @@ +--- +title: Custom Domain in Data Service +summary: Learn how to use a custom domain to access your Data App in TiDB Cloud Data Service. +--- + +# Custom Domain in Data Service + +TiDB Cloud Data Service provides a default domain `.data.tidbcloud.com` to access each Data App's endpoints. For enhanced personalization and flexibility, you can configure a custom domain for your Data App instead of using the default domain. + +This document describes how to manage custom domains in your Data App. + +## Before you begin + +Before configuring a custom domain for your Data App, note the following: + +- Custom domain requests exclusively support HTTPS for security. Once you successfully configure a custom domain, a "Let's Encrypt" certificate is automatically applied. +- Your custom domain must be unique within the TiDB Cloud Data Service. +- You can configure only one custom domain for each default domain, which is determined by the region of your cluster. + +## Manage custom domains + +The following sections describe how to create, edit, and remove a custom domain for a Data App. + +### Create a custom domain + +To create a custom domain for a Data App, perform the following steps: + +1. Navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project. +2. In the left pane, click the name of your target Data App to view its details. +3. In the **Manage Custom Domain** area, click **Add Custom Domain**. +4. In the **Add Custom Domain** dialog box, do the following: + 1. Select the default domain you want to replace. + 2. Enter your desired custom domain name. + 3. Optional: configure a custom path as the prefix for your endpoints. If **Custom Path** is left empty, the default path is used. +5. Preview your **Base URL** to ensure it meets your expectations. If it looks correct, click **Save**. +6. Follow the instructions in the **DNS Settings** dialog to add a `CNAME` record for the default domain in your DNS provider. + +The custom domain is in a **Pending** status initially while the system validates your DNS settings. Once the DNS validation is successful, the status of your custom domain will update to **Success**. + +> **Note:** +> +> Depending on your DNS provider, it might take up to 24 hours for the DNS record to be validated. If a custom domain remains unvalidated for over 24 hours, it will be in an **Expired** status. In this case, you can only remove the custom domain and try again. + +After your custom domain status is set to **Success**, you can use it to access your endpoint. The code example provided by TiDB Cloud Data Service is automatically updated to your custom domain and path. For more information, see [Call an endpoint](/tidb-cloud/data-service-manage-endpoint.md#call-an-endpoint). + +### Edit a custom domain + +> **Note:** +> +> After you complete the following changes, the previous custom domain and custom path will become invalid immediately. If you modify the custom domain, you need to wait for the new DNS record to be validated. + +To edit a custom domain for a Data App, perform the following steps: + +1. Navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project. +2. In the left pane, click the name of your target Data App to view its details. +3. In the **Manage Custom Domain** area, locate the **Action** column, and then click **Edit** in the custom domain row that you want to edit. +4. In the displayed dialog box, update the custom domain or custom path. +5. Preview your **Base URL** to ensure it meets your expectations. If it looks correct, click **Save**. +6. If you have changed the custom domain, follow the instructions in the **DNS Settings** dialog to add a `CNAME` record for the default domain in your DNS provider. + +### Remove a custom domain + +> **Note:** +> +> Before you delete a custom domain, make sure that the custom domain is not used anymore. + +To remove a custom domain for a Data App, perform the following steps: + +1. Navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project. +2. In the left pane, click the name of your target Data App to view its details. +3. In the **Manage Custom Domain** area, locate the **Action** column, and then click **Delete** in the custom domain row that you want to delete. +4. In the displayed dialog box, confirm the deletion. diff --git a/tidb-cloud/data-service-get-started.md b/tidb-cloud/data-service-get-started.md index 73d613373e49d..0733b2e7a0804 100644 --- a/tidb-cloud/data-service-get-started.md +++ b/tidb-cloud/data-service-get-started.md @@ -13,26 +13,58 @@ Data Service (beta) enables you to access TiDB Cloud data via an HTTPS request u > > For more information, see [Get started with Chat2Query API](/tidb-cloud/use-chat2query-api.md). -This document introduces how to quickly get started with TiDB Cloud Data Service (beta) by creating a Data App, developing, testing, deploying, and calling an endpoint. +This document introduces how to quickly get started with TiDB Cloud Data Service (beta) by creating a Data App, developing, testing, deploying, and calling an endpoint. A Data App is a collection of endpoints that you can use to access data for a specific application. ## Before you begin -Before creating a Data App, make sure that you have created a [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster. If you do not have one, follow the steps in [Create a TiDB Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create one. +Before creating a Data App, make sure that you have created a [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) cluster. If you do not have one, follow the steps in [Create a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create one. -## Step 1. Create a Data App +## Get started with a sample Data App -A Data App is a collection of endpoints that you can use to access data for a specific application. To create a Data App, perform the following steps: +Creating a sample Data App is the best way to get started with Data Service. If your project does not have any Data App yet, you can follow the on-screen instructions on the **Data Service** page to create a sample Data App and use this App to explore Data Service features. 1. In the [TiDB Cloud console](https://tidbcloud.com), click **Data Service** in the left navigation pane. -2. On the **Data Service** page, click **Create Data App**. +2. On the **Data Service** page, click **Create Sample Data App**. A dialog is displayed. + +3. In the dialog, update the App name if necessary, select clusters that you want the Data App to access, and then click **Create**. + + The creation process takes a few seconds. + + > **Note:** + > + > If there is no cluster in your current project, you can click **Create New Cluster** in the **Link Data Sources** drop-down list to create one first. + +4. After the sample Data App is automatically created, you can find the App name, a list of endpoints in the left pane, the SQL statements of an endpoint in the middle pane, and instructions about using the sample Data App on the right side. + +5. Follow the instructions on the right side to choose an endpoint and use the curl command to call the endpoint. + +## Get started with your own Data App + +To get started with Data Service, you can also create your own Data App, and then develop, test, deploy, and call endpoints according to the following steps. + +### Step 1. Create a Data App + + To create a Data App, perform the following steps: + +1. In the [TiDB Cloud console](https://tidbcloud.com), click **Data Service** in the left navigation pane. + +2. On the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project, click **Create DataApp** in the left pane. + + > **Tip:** + > + > If this is the first Data App in your project, click **Create Data App** in the middle of the page. 3. In the **Create Data App** dialog, enter a name, a description, and select clusters that you want the Data App to access. + > **Note:** + > + > By default, the Data App type is **Standard Data App**. If you want to create a **Chat2Query Data App**, refer to [Get Started with Chat2Query API](/tidb-cloud/use-chat2query-api.md) instead of this document. + 4. (Optional) To automatically deploy endpoints of the Data App to your preferred GitHub repository and branch, enable **Connect to GitHub**, and then do the following: 1. Click **Install on GitHub**, and then follow the on-screen instructions to install **TiDB Cloud Data Service** as an application on your target repository. - 2. Click **Authorize** to authorize access to the application on GitHub. + 2. Go back to the TiDB Cloud console, and then click **Authorize** to authorize access to the application on GitHub. 3. Specify the target repository, branch, and directory where you want to save the configuration files of your Data App. > **Note:** @@ -46,15 +78,13 @@ A Data App is a collection of endpoints that you can use to access data for a sp For your new Data App, **Auto Sync & Deployment** and **Review Draft** are enabled by default so you can easily synchronize Data App changes between TiDB Cloud console and GitHub and review changes before the deployment. For more information about the GitHub integration, see [Deploy your Data App changes with GitHub automatically](/tidb-cloud/data-service-manage-github-connection.md). -## Step 2. Develop an endpoint +### Step 2. Develop an endpoint An endpoint is a web API that you can customize to execute SQL statements. -After creating a Data App, a default `untitled endpoint` is created for you automatically. You can use the default endpoint to access your TiDB Cloud cluster. - -If you want to create a new endpoint, locate the newly created Data App and click **+** **Create Endpoint** to the right of the App name. +To create a new endpoint, locate the newly created Data App and click **+** **Create Endpoint** to the right of the App name. -### Configure properties +#### Configure properties On the right pane, click the **Properties** tab and set properties for the endpoint, such as: @@ -66,7 +96,7 @@ On the right pane, click the **Properties** tab and set properties for the endpo For more information about endpoint properties, see [Configure properties](/tidb-cloud/data-service-manage-endpoint.md#configure-properties). -### Write SQL statements +#### Write SQL statements You can customize SQL statements for the endpoint in the SQL editor, which is the middle pane on the **Data Service** page. @@ -84,6 +114,10 @@ You can customize SQL statements for the endpoint in the SQL editor, which is th In the SQL editor, you can write statements such as table join queries, complex queries, and aggregate functions. You can also simply type `--` followed by your instructions to let AI generate SQL statements automatically. + > **Note:** + > + > To try the AI capacity of TiDB Cloud, you need to allow PingCAP and OpenAI to use your code snippets for research and service improvement. For more information, see [Enable or disable AI to generate SQL queries](/tidb-cloud/explore-data-with-chat2query.md#enable-or-disable-ai-to-generate-sql-queries). + To define a parameter, you can insert it as a variable placeholder like `${ID}` in the SQL statement. For example, `SELECT * FROM table_name WHERE id = ${ID}`. Then, you can click the **Params** tab on the right pane to change the parameter definition and test values. > **Note:** @@ -91,7 +125,7 @@ You can customize SQL statements for the endpoint in the SQL editor, which is th > - The parameter name is case-sensitive. > - The parameter cannot be used as a table name or column name. - - In the **Definition** section, you can specify whether the parameter is required when a client calls the endpoint, the data type (`STRING`, `NUMBER`, `INTEGER`, or `BOOLEAN`), and the default value of the parameter. When using a `STRING` type parameter, you do not need to add quotation marks (`'` or `"`). For example, `foo` is valid for the `STRING` type and is processed as `"foo"`, whereas `"foo"` is processed as `"\"foo\""`. + - In the **Definition** section, you can specify whether the parameter is required when a client calls the endpoint, the data type, and the default value of the parameter. - In the **Test Values** section, you can set the test value for a parameter. The test values are used when you run the SQL statements or test the endpoint. If you do not set the test values, the default values are used. - For more information, see [Configure parameters](/tidb-cloud/data-service-manage-endpoint.md#configure-parameters). @@ -99,13 +133,35 @@ You can customize SQL statements for the endpoint in the SQL editor, which is th If you have inserted parameters in the SQL statements, make sure that you have set test values or default values for the parameters in the **Params** tab on the right pane. Otherwise, an error is returned. - To run a SQL statement, select the line of the SQL with your cursor and click **Run** > **Run at cursor**. + +
    + + For macOS: + + - If you have only one statement in the editor, to run it, press **⌘ + Enter** or click **Run**. + + - If you have multiple statements in the editor, to run one or several of them sequentially, place your cursor on your target statement or select the lines of the target statements with your cursor, and then press **⌘ + Enter** or click **Run**. + + - To run all statements in the editor sequentially, press **⇧ + ⌘ + Enter**, or select the lines of all statements with your cursor and click **Run**. + +
    + +
    + + For Windows or Linux: + + - If you have only one statement in the editor, to run it, press **Ctrl + Enter** or click **Run**. + + - If you have multiple statements in the editor, to run one or several of them sequentially, place your cursor on your target statement or select the lines of the target statements with your cursor, and then press **Ctrl + Enter** or click **Run**. + + - To run all statements in the editor sequentially, press **Shift + Ctrl + Enter**, or select the lines of all statements with your cursor and click **Run**. - To run all SQL statements in the SQL editor, click **Run**. In this case, only the last SQL results are returned. +
    +
    After running the statements, you can see the query results immediately in the **Result** tab at the bottom of the page. -## Step 3. Test the endpoint (optional) +### Step 3. Test the endpoint (optional) After configuring an endpoint, you can test the endpoint to verify whether it works as expected before deploying. @@ -113,7 +169,7 @@ To test the endpoint, click **Test** in the upper-right corner or press **F5**. Then, you can see the response in the **HTTP Response** tab at the bottom of the page. For more information about the response, see [Response of an endpoint](/tidb-cloud/data-service-manage-endpoint.md#response). -## Step 4. Deploy the endpoint +### Step 4. Deploy the endpoint To deploy the endpoint, perform the following steps: @@ -121,22 +177,27 @@ To deploy the endpoint, perform the following steps: 2. Click **Deploy** to confirm the deployment. You will get the **Endpoint has been deployed** prompt if the endpoint is successfully deployed. - On the right pane of the endpoint details page, you can click the **Deployments** tab to view the deployed history. + To view the deployment history, you can click the name of your Data App in the left pane, and then click the **Deployments** tab in the right pane. -## Step 5. Call the endpoint +### Step 5. Call the endpoint You can call the endpoint by sending an HTTPS request. Before calling an endpoint, you need to first obtain an API key for the Data App. -### 1. Create an API key +#### 1. Create an API key 1. In the left pane of the [**Data Service**](https://tidbcloud.com/console/data-service) page, click the name of your Data App to view its details. 2. In the **Authentication** area, click **Create API Key**. -3. In the **Create API Key** dialog box, enter a description and select a role for your API key. +3. In the **Create API Key** dialog box, do the following: + + 1. (Optional) Enter a description for your API key. + 2. Select a role for your API key. - The role is used to control whether the API key can read or write data to the clusters linked to the Data App. You can select the `ReadOnly` or `ReadAndWrite` role: + The role is used to control whether the API key can read or write data to the clusters linked to the Data App. You can select the `ReadOnly` or `ReadAndWrite` role: - - `ReadOnly`: only allows the API key to read data, such as `SELECT`, `SHOW`, `USE`, `DESC`, and `EXPLAIN` statements. - - `ReadAndWrite`: allows the API key to read and write data. You can use this API key to execute all SQL statements, such as DML and DDL statements. + - `ReadOnly`: only allows the API key to read data, such as `SELECT`, `SHOW`, `USE`, `DESC`, and `EXPLAIN` statements. + - `ReadAndWrite`: allows the API key to read and write data. You can use this API key to execute all SQL statements, such as DML and DDL statements. + + 3. (Optional) Set a desired rate limit for your API key. 4. Click **Next**. The public key and private key are displayed. @@ -144,7 +205,9 @@ You can call the endpoint by sending an HTTPS request. Before calling an endpoin 5. Click **Done**. -### 2. Get the code example +For more information about API keys, see [API Keys in Data Service](/tidb-cloud/data-service-api-key.md). + +#### 2. Get the code example TiDB Cloud generates code examples to help you call an endpoint. To get the code example, perform the following steps: @@ -186,7 +249,7 @@ TiDB Cloud generates code examples to help you call an endpoint. To get the code > - By requesting the regional domain `.data.tidbcloud.com`, you can directly access the endpoint in the region where the TiDB cluster is located. > - Alternatively, you can also request the global domain `data.tidbcloud.com` without specifying a region. In this way, TiDB Cloud will internally redirect the request to the target region, but this might result in additional latency. If you choose this way, make sure to add the `--location-trusted` option to your curl command when calling an endpoint. -### 3. Use the code example +#### 3. Use the code example Paste the code example in your application and run it. Then, you can get the response of the endpoint. diff --git a/tidb-cloud/data-service-integrations.md b/tidb-cloud/data-service-integrations.md new file mode 100644 index 0000000000000..32b6ddc0f72a7 --- /dev/null +++ b/tidb-cloud/data-service-integrations.md @@ -0,0 +1,40 @@ +--- +title: Integrate a Data App with Third-Party Tools +summary: Learn how to integrate a TiDB Cloud Data App with third-party tools, such as GPTs and Dify, in the TiDB Cloud console. +--- + +# Integrate a Data App with Third-Party Tools + +Integrating third-party tools with your Data App enhances your applications with advanced natural language processing and artificial intelligence (AI) capabilities provided by third-party tools. This integration enables your applications to perform more complex tasks and deliver intelligent solutions. + +This document describes how to integrate a Data App with third-party tools, such as GPTs and Dify, in the TiDB Cloud console. + +## Integrate your Data App with GPTs + +You can integrate your Data App with [GPTs](https://openai.com/blog/introducing-gpts) to enhance your applications with intelligent capabilities. + +To integrate your Data App with GPTs, perform the following steps: + +1. Navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project. +2. In the left pane, locate your target Data App, click the name of your target Data App, and then click the **Integrations** tab. +3. In the **Integrate with GPTs** area, click **Get Configuration**. + + ![Get Configuration](/media/tidb-cloud/data-service/GPTs1.png) + +4. In the displayed dialog box, you can see the following fields: + + a. **API Specification URL**: copy the URL of the OpenAPI Specification of your Data App. For more information, see [Use the OpenAPI Specification](/tidb-cloud/data-service-manage-data-app.md#use-the-openapi-specification). + + b. **API Key**: enter the API key of your Data App. If you do not have an API key yet, click **Create API Key** to create one. For more information, see [Create an API key](/tidb-cloud/data-service-api-key.md#create-an-api-key). + + c. **API Key Encoded**: copy the base64 encoded string equivalent to the API key you have provided. + + ![GPTs Dialog Box](/media/tidb-cloud/data-service/GPTs2.png) + +5. Use the copied API Specification URL and the encoded API key in your GPT configuration. + +## Integrate your Data App with Dify + +You can integrate your Data App with [Dify](https://docs.dify.ai/guides/tools) to enhance your applications with intelligent capabilities, such as vector distance calculations, advanced similarity searches, and vector analysis. + +To integrate your Data App with Dify, follow the same steps as for [GPTs integration](#integrate-your-data-app-with-gpts). The only difference is that on the **Integrations** tab, you need to click **Get Configuration** in the **Integrate with Dify** area. diff --git a/tidb-cloud/data-service-manage-data-app.md b/tidb-cloud/data-service-manage-data-app.md index 06d60f61d529b..962dc11193490 100644 --- a/tidb-cloud/data-service-manage-data-app.md +++ b/tidb-cloud/data-service-manage-data-app.md @@ -21,6 +21,10 @@ To create a Data App for your project, perform the following steps: 2. Enter a name, a description, and select clusters that you want the Data App to access. + > **Note:** + > + > By default, the Data App type is **Standard Data App**. If you want to create a **Chat2Query Data App**, refer to [Get Started with Chat2Query API](/tidb-cloud/use-chat2query-api.md) instead of this document. + 3. (Optional) To automatically deploy endpoints of the Data App to your preferred GitHub repository and branch, enable **Connect to GitHub**, and then do the following: 1. Click **Install on GitHub**, and then follow the on-screen instructions to install **TiDB Cloud Data Service** as an application on your target repository. @@ -85,6 +89,10 @@ For more information, see [Manage an API key](/tidb-cloud/data-service-api-key.m For more information, see [Manage an endpoint](/tidb-cloud/data-service-manage-endpoint.md). +### Manage a custom domain + +For more information, see [Manage a custom domain](/tidb-cloud/data-service-custom-domain.md). + ### Manage deployments To manage deployments, perform the following steps: @@ -167,4 +175,4 @@ To delete a Data App, perform the following steps: ## Learn more -- [Run Data App in Postman](/tidb-cloud/data-service-postman-integration.md) \ No newline at end of file +- [Run Data App in Postman](/tidb-cloud/data-service-postman-integration.md) diff --git a/tidb-cloud/data-service-manage-endpoint.md b/tidb-cloud/data-service-manage-endpoint.md index 4acc03f2ecd80..b0e154c8e38ea 100644 --- a/tidb-cloud/data-service-manage-endpoint.md +++ b/tidb-cloud/data-service-manage-endpoint.md @@ -20,11 +20,11 @@ This document describes how to manage your endpoints in a Data App in the TiDB C ## Create an endpoint -In Data Service, you can either generate an endpoint automatically or create an endpoint manually. +In Data Service, you can automatically generate endpoints, manually create endpoints, or add predefined system endpoints. > **Tip:** > -> You can also create an endpoint from a SQL file in Chat2Query (beta). For more information, see [Generate an endpoint from a SQL file](/tidb-cloud/explore-data-with-chat2query.md#generate-an-endpoint-from-a-sql-file). +> You can also create an endpoint from a SQL file in SQL Editor. For more information, see [Generate an endpoint from a SQL file](/tidb-cloud/explore-data-with-chat2query.md#generate-an-endpoint-from-a-sql-file). ### Generate an endpoint automatically @@ -40,10 +40,17 @@ In TiDB Cloud Data Service, you can generate one or multiple endpoints automatic > > The **Table** drop-down list includes only user-defined tables with at least one column, excluding system tables and any tables without a column definition. - 2. Select at least one HTTP operation (such as `GET Retrieve`, `POST Create`, and `PUT Update`) for the endpoint to be generated. + 2. Select at least one HTTP operation (such as `GET (Retrieve)`, `POST (Create)`, and `PUT (Update)`) for the endpoint to be generated. - For each operation you selected, TiDB Cloud Data Service will generate a corresponding endpoint. If you have selected a batch operation (such as `POST Batch Create`), the generated endpoint lets you operate on multiple rows in a single request. + For each operation you select, TiDB Cloud Data Service will generate a corresponding endpoint. If you select a batch operation (such as `POST (Batch Create)`), the generated endpoint lets you operate on multiple rows in a single request. + If the table you selected contains [vector data types](/tidb-cloud/vector-search-data-types.md), you can enable the **Vector Search Operations** option and select a vector distance function to generate a vector search endpoint that automatically calculates vector distances based on your selected distance function. The supported [vector distance functions](/tidb-cloud/vector-search-functions-and-operators.md) include the following: + + - `VEC_L2_DISTANCE` (default): calculates the L2 distance (Euclidean distance) between two vectors. + - `VEC_COSINE_DISTANCE`: calculates the cosine distance between two vectors. + - `VEC_NEGATIVE_INNER_PRODUCT`: calculates the distance by using the negative of the inner product between two vectors. + - `VEC_L1_DISTANCE`: calculates the L1 distance (Manhattan distance) between two vectors. + 3. (Optional) Configure a timeout and tag for the operations. All the generated endpoints will automatically inherit the configured properties, which can be modified later as needed. 4. (Optional) The **Auto-Deploy Endpoint** option (disabled by default) controls whether to enable the direct deployment of the generated endpoints. When it is enabled, the draft review process is skipped, and the generated endpoints are deployed immediately without further manual review or approval. @@ -53,10 +60,11 @@ In TiDB Cloud Data Service, you can generate one or multiple endpoints automatic 5. Check the generated endpoint name, SQL statements, properties, and parameters of the new endpoint. - - Endpoint name: the generated endpoint name is the name of the selected table, and the request method (such as `GET`, `POST`, and `PUT`) is displayed before the name. For example, if the selected table name is `sample-table` and the selected operation is **POST Create**, the generated endpoint is displayed as `POST sample-table`. + - Endpoint name: the generated endpoint name is in the `/` format, and the request method (such as `GET`, `POST`, and `PUT`) is displayed before the endpoint name. For example, if the selected table name is `sample_table` and the selected operation is `POST (Create)`, the generated endpoint is displayed as `POST /sample_table`. - - If a batch operation is selected, TiDB Cloud Data Service appends `_batch` to the name of the generated endpoint. For example, if the selected table name is `sample-table` and the selected operation is **POST Batch Create**, the generated endpoint is displayed as `POST sample-table_batch`. - - If there has been already an endpoint with the same request method and endpoint name, TiDB Cloud Data Service appends `_copy` to the name of the generated endpoint. For example, `sample-table_copy`. + - If a batch operation is selected, TiDB Cloud Data Service appends `/bulk` to the name of the generated endpoint. For example, if the selected table name is `/sample_table` and the selected operation is `POST (Batch Create)`, the generated endpoint is displayed as `POST /sample_table/bulk`. + - If `POST (Vector Similarity Search)` is selected, TiDB Cloud Data Service appends `/vector_search` to the name of the generated endpoint. For example, if the selected table name is `/sample_table` and the selected operation is `POST (Vector Similarity Search)`, the generated endpoint is displayed as `POST /sample_table/vector_search`. + - If there has been already an endpoint with the same request method and endpoint name, TiDB Cloud Data Service appends `_dump_` to the name of the generated endpoint. For example, `/sample_table_dump_EUKRfl`. - SQL statements: TiDB Cloud Data Service automatically writes SQL statements for the generated endpoints according to the table column specifications and the selected endpoint operations. You can click the endpoint name to view its SQL statements in the middle section of the page. - Endpoint properties: TiDB Cloud Data Service automatically configures the endpoint path, request method, timeout, and tag according to your selection. You can find the properties in the right pane of the page. @@ -73,6 +81,44 @@ To create an endpoint manually, perform the following steps: 3. Update the default name if necessary. The newly created endpoint is added to the top of the endpoint list. 4. Configure the new endpoint according to the instructions in [Develop an endpoint](#develop-an-endpoint). +### Add a predefined system endpoint + +Data Service provides an endpoint library with predefined system endpoints that you can directly add to your Data App, reducing the effort in your endpoint development. Currently, the library only includes the `/system/query` endpoint, which enables you to execute any SQL statement by simply passing the statement in the predefined `sql` parameter. + +To add a predefined system endpoint to your Data App, perform the following steps: + +1. Navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project. + +2. In the left pane, locate your target Data App, click **+** to the right of the App name, and then click **Manage Endpoint Library**. + + A dialog for endpoint library management is displayed. Currently, only **Execute Query** (that is, the `/system/query` endpoint) is provided in the dialog. + +3. To add the `/system/query` endpoint to your Data App, toggle the **Execute Query** switch to **Added**. + + > **Tip:** + > + > To remove an added predefined endpoint from your Data App, toggle the **Execute Query** switch to **Removed**. + +4. Click **Save**. + + > **Note:** + > + > - After you click **Save**, the added or removed endpoint is deployed to production immediately, which makes the added endpoint accessible and the removed endpoint inaccessible immediately. + > - If a non-predefined endpoint with the same path and method already exists in the current App, the creation of the system endpoint will fail. + + The added system-provided endpoint is displayed at the top of the endpoint list. + +5. Check the endpoint name, SQL statements, properties, and parameters of the new endpoint. + + > **Note:** + > + > The `/system/query` endpoint is powerful and versatile but can be potentially destructive. Use it with discretion and ensure the queries are secure and well-considered to prevent unintended consequences. + + - Endpoint name: the endpoint name and path is `/system/query`, and the request method `POST`. + - SQL statements: the `/system/query` endpoint does not come with any SQL statement. You can find the SQL editor in the middle section of the page and write your desired SQL statements in the SQL editor. Note that the SQL statements written in the SQL editor for the `/system/query` endpoint will be saved in the SQL editor so you can further develop and test them next time but they will not be saved in the endpoint configuration. + - Endpoint properties: in the right pane of the page, you can find the endpoint properties on the **Properties** tab. Unlike other custom endpoints, only the `timeout` and `max rows` properties can be customized for system endpoints. + - Endpoint parameters: in the right pane of the page, you can find the endpoint parameters on the **Params** tab. The parameters of the `/system/query` endpoint are configured automatically and cannot be modified. + ## Develop an endpoint For each endpoint, you can write SQL statements to execute on a TiDB cluster, define parameters for the SQL statements, or manage the name and version. @@ -89,11 +135,38 @@ On the right pane of the endpoint details page, you can click the **Properties** - **Path**: the path that users use to access the endpoint. - - The combination of the request method and the path must be unique within a Data App. - - Only letters, numbers, underscores (`_`), and slashes (`/`) are allowed in a path. A path must start with a slash (`/`) and end with a letter, number, or underscore (`_`). For example, `/my_endpoint/get_id`. - The length of the path must be less than 64 characters. + - The combination of the request method and the path must be unique within a Data App. + - Only letters, numbers, underscores (`_`), slashes (`/`), and parameters enclosed in curly braces (such as `{var}`) are allowed in a path. Each path must start with a slash (`/`) and end with a letter, number, or underscore (`_`). For example, `/my_endpoint/get_id`. + - For parameters enclosed in `{ }`, only letters, numbers, and underscores (`_`) are allowed. Each parameter enclosed in `{ }` must start with a letter or underscore (`_`). + + > **Note:** + > + > - In a path, each parameter must be at a separate level and does not support prefixes or suffixes. + > + > Valid path: ```/var/{var}``` and ```/{var}``` + > + > Invalid path: ```/var{var}``` and ```/{var}var``` + > + > - Paths with the same method and prefix might conflict, as in the following example: + > + > ```GET /var/{var1}``` + > + > ```GET /var/{var2}``` + > + > These two paths will conflict with each other because `GET /var/123` matches both. + > + > - Paths with parameters have lower priority than paths without parameters. For example: + > + > ```GET /var/{var1}``` + > + > ```GET /var/123``` + > + > These two paths will not conflict because `GET /var/123` takes precedence. + > + > - Path parameters can be used directly in SQL. For more information, see [Configure parameters](#configure-parameters). -- **Endpoint URL**: (read-only) the URL is automatically generated based on the region where the corresponding cluster is located, the service URL of the Data App, and the path of the endpoint. For example, if the path of the endpoint is `/my_endpoint/get_id`, the endpoint URL is `https://.data.tidbcloud.com/api/v1beta/app//endpoint/my_endpoint/get_id`. +- **Endpoint URL**: (read-only) the default URL is automatically generated based on the region where the corresponding cluster is located, the service URL of the Data App, and the path of the endpoint. For example, if the path of the endpoint is `/my_endpoint/get_id`, the endpoint URL is `https://.data.tidbcloud.com/api/v1beta/app//endpoint/my_endpoint/get_id`. To configure a custom domain for the Data App, see [Custom Domain in Data Service](/tidb-cloud/data-service-custom-domain.md). - **Request Method**: the HTTP method of the endpoint. The following methods are supported: @@ -117,8 +190,12 @@ On the right pane of the endpoint details page, you can click the **Properties** > - The `page_size` must be less than or equal to the **Max Rows** property. Otherwise, an error is returned. - **Cache Response**: this property is available only when the request method is `GET`. When **Cache Response** is enabled, TiDB Cloud Data Service can cache the response returned by your `GET` requests within a specified time-to-live (TTL) period. -- **Time-to-live (s)**: this property is available only when **Cache Response** is enabled. You can use it to specify the time-to-live (TTL) period in seconds for cached response. During the TTL period, if you make the same `GET` requests again, Data Service returns the cached response directly instead of fetching data from the target database again, which improves your query performance. -- **Batch Operation**: this property is visible only when the request method is `POST`, `PUT`, or `DELETE`. When **Batch Operation** is enabled, you can operate on multiple rows in a single request. For example, you can insert multiple rows of data in a single `POST` request by adding an array of data objects in the `--data-raw` option of your curl command when calling the endpoint. +- **Time-to-live(s)**: this property is available only when **Cache Response** is enabled. You can use it to specify the time-to-live (TTL) period in seconds for cached response. During the TTL period, if you make the same `GET` requests again, Data Service returns the cached response directly instead of fetching data from the target database again, which improves your query performance. +- **Batch Operation**: this property is visible only when the request method is `POST` or `PUT`. When **Batch Operation** is enabled, you can operate on multiple rows in a single request. For example, you can insert multiple rows of data in a single `POST` request by putting an array of data objects to the `items` field of an object in the `--data-raw` option of your curl command when [calling the endpoint](#call-an-endpoint). + + > **Note:** + > + > The endpoint with **Batch Operation** enabled supports both array and object formats for the request body: `[{dataObject1}, {dataObject2}]` and `{items: [{dataObject1}, {dataObject2}]}`. For better compatibility with other systems, it is recommended that you use the object format `{items: [{dataObject1}, {dataObject2}]}`. ### Write SQL statements @@ -132,29 +209,60 @@ On the SQL editor of the endpoint details page, you can write and run the SQL st On the upper part of the SQL editor, select a cluster on which you want to execute SQL statements from the drop-down list. Then, you can view all databases of this cluster in the **Schema** tab on the right pane. -2. Write SQL statements. +2. Depending on your endpoint type, do one of the following to select a database: + + - Predefined system endpoints: on the upper part of the SQL editor, select the target database from the drop-down list. + - Other endpoints: write a SQL statement to specify the target database in the SQL editor. For example, `USE database_name;`. - Before querying or modifying data, you need to first specify the database in the SQL statements. For example, `USE database_name;`. +3. Write SQL statements. In the SQL editor, you can write statements such as table join queries, complex queries, and aggregate functions. You can also simply type `--` followed by your instructions to let AI generate SQL statements automatically. To define a parameter, you can insert it as a variable placeholder like `${ID}` in the SQL statement. For example, `SELECT * FROM table_name WHERE id = ${ID}`. Then, you can click the **Params** tab on the right pane to change the parameter definition and test values. For more information, see [Parameters](#configure-parameters). + When defining an array parameter, the parameter is automatically converted to multiple comma-separated values in the SQL statement. To make sure that the SQL statement is valid, you need to add parentheses (`()`) around the parameter in some SQL statements (such as `IN`). For example, if you define an array parameter `ID` with test value `1,2,3`, use `SELECT * FROM table_name WHERE id IN (${ID})` to query the data. + > **Note:** > > - The parameter name is case-sensitive. > - The parameter cannot be used as a table name or column name. -3. Run SQL statements. +4. Run SQL statements. If you have inserted parameters in the SQL statements, make sure that you have set test values or default values for the parameters in the **Params** tab on the right pane. Otherwise, an error is returned. - To run a SQL statement, select the line of the SQL with your cursor and click **Run** > **Run at cursor**. + +
    + + For macOS: + + - If you have only one statement in the editor, to run it, press **⌘ + Enter** or click **Run**. + + - If you have multiple statements in the editor, to run one or several of them sequentially, place your cursor on your target statement or select the lines of the target statements with your cursor, and then press **⌘ + Enter** or click **Run**. + + - To run all statements in the editor sequentially, press **⇧ + ⌘ + Enter**, or select the lines of all statements with your cursor and click **Run**. + +
    + +
    + + For Windows or Linux: + + - If you have only one statement in the editor, to run it, press **Ctrl + Enter** or click **Run**. - To run all SQL statements in the SQL editor, click **Run**. In this case, only the last SQL results are returned. + - If you have multiple statements in the editor, to run one or several of them sequentially, place your cursor on your target statement or select the lines of the target statements with your cursor, and then press **Ctrl + Enter** or click **Run**. + + - To run all statements in the editor sequentially, press **Shift + Ctrl + Enter**, or select the lines of all statements with your cursor and click **Run**. + +
    +
    After running the statements, you can see the query results immediately in the **Result** tab at the bottom of the page. + > **Note:** + > + > The returned result has a size limit of 8 MiB. + ### Configure parameters On the right pane of the endpoint details page, you can click the **Params** tab to view and manage the parameters used in the endpoint. @@ -162,21 +270,28 @@ On the right pane of the endpoint details page, you can click the **Params** tab In the **Definition** section, you can view and manage the following properties for a parameter: - The parameter name: the name can only include letters, digits, and underscores (`_`) and must start with a letter or an underscore (`_`). **DO NOT** use `page` and `page_size` as parameter names, which are reserved for pagination of request results. -- **Required**: specifies whether the parameter is required in the request. The default configuration is set to not required. -- **Type**: specifies the data type of the parameter. Supported values are `STRING`, `NUMBER`, `INTEGER`, and `BOOLEAN`. When using a `STRING` type parameter, you do not need to add quotation marks (`'` or `"`). For example, `foo` is valid for the `STRING` type and is processed as `"foo"`, whereas `"foo"` is processed as `"\"foo\""`. +- **Required**: specifies whether the parameter is required in the request. For path parameters, the configuration is required and cannot be modified. For other parameters, the default configuration is not required. +- **Type**: specifies the data type of the parameter. For path parameters, only `STRING` and `INTEGER` are supported. For other parameters, `STRING`, `NUMBER`, `INTEGER`, `BOOLEAN`, and `ARRAY` are supported. + + When using a `STRING` type parameter, you do not need to add quotation marks (`'` or `"`). For example, `foo` is valid for the `STRING` type and is processed as `"foo"`, whereas `"foo"` is processed as `"\"foo\""`. + +- **Enum Value**: (optional) specifies the valid values for the parameter and is available only when the parameter type is `STRING`, `INTEGER`, or `NUMBER`. + + - If you leave this field empty, the parameter can be any value of the specified type. + - To specify multiple valid values, you can separate them with a comma (`,`). For example, if you set the parameter type to `STRING` and specify this field as `foo, bar`, the parameter value can only be `foo` or `bar`. + +- **ItemType**: specifies the item type of an `ARRAY` type parameter. - **Default Value**: specifies the default value of the parameter. + - For `ARRAY` type, you need to separate multiple values with a comma (`,`). - Make sure that the value can be converted to the type of parameter. Otherwise, the endpoint returns an error. - If you do not set a test value for a parameter, the default value is used when testing the endpoint. +- **Location**: indicates the location of the parameter. This property cannot be modified. + - For path parameters, this property is `Path`. + - For other parameters, if the request method is `GET` or `DELETE`, this property is `Query`. If the request method is `POST` or `PUT`, this property is `Body`. In the **Test Values** section, you can view and set test parameters. These values are used as the parameter values when you test the endpoint. Make sure that the value can be converted to the type of parameter. Otherwise, the endpoint returns an error. -### Manage versions - -On the right pane of the endpoint details page, you can click the **Deployments** tab to view and manage the deployed versions of the endpoint. - -In the **Deployments** tab, you can deploy a draft version and undeploy the online version. - ### Rename To rename an endpoint, perform the following steps: @@ -185,6 +300,10 @@ To rename an endpoint, perform the following steps: 2. In the left pane, click the name of your target Data App to view its endpoints. 3. Locate the endpoint you want to rename, click **...** > **Rename**., and enter a new name for the endpoint. +> **Note:** +> +> Predefined system endpoints do not support renaming. + ## Test an endpoint To test an endpoint, perform the following steps: @@ -274,13 +393,17 @@ TiDB Cloud Data Service generates code examples to help you call an endpoint. To ```bash curl --digest --user ':' \ - --request POST 'https://.data.tidbcloud.com/api/v1beta/app//endpoint/' \ - --header 'content-type: application/json'\ - --header 'endpoint-type: draft' - --data-raw '[{ - "age": "${age}", - "career": "${career}" - }]' + --request POST 'https://.data.tidbcloud.com/api/v1beta/app//endpoint/' \ + --header 'content-type: application/json'\ + --header 'endpoint-type: draft' + --data-raw '{ + "items": [ + { + "age": "${age}", + "career": "${career}" + } + ] + }' ```
    @@ -293,12 +416,16 @@ TiDB Cloud Data Service generates code examples to help you call an endpoint. To ```bash curl --digest --user ':' \ - --request POST 'https://.data.tidbcloud.com/api/v1beta/app//endpoint/' - --header 'content-type: application/json'\ - --data-raw '[{ - "age": "${age}", - "career": "${career}" - }]' + --request POST 'https://.data.tidbcloud.com/api/v1beta/app//endpoint/' \ + --header 'content-type: application/json'\ + --data-raw '{ + "items": [ + { + "age": "${age}", + "career": "${career}" + } + ] + }' ```
    @@ -315,10 +442,9 @@ TiDB Cloud Data Service generates code examples to help you call an endpoint. To - If the request method of your endpoint is `GET` and **Pagination** is enabled for the endpoint, you can paginate the results by updating the values of `page=` and `page_size=` with your desired values. For example, to get the second page with 10 items per page, use `page=2` and `page_size=10`. - If the request method of your endpoint is `POST` or `PUT`, fill in the `--data-raw` option according to the rows of data that you want to operate on. - - For endpoints with **Batch Operation** enabled, the `--data-raw` option accepts an array of data objects so you can operate on multiple rows of data using one endpoint. + - For endpoints with **Batch Operation** enabled, the `--data-raw` option accepts an object with an `items` field containing an array of data objects so you can operate on multiple rows of data using one endpoint. - For endpoints with **Batch Operation** not enabled, the `--data-raw` option only accepts one data object. - - If the request method of your endpoint is `DELETE` and **Batch Operation** is enabled for the endpoint, you can use comma (`,`) to separate multiple rows to be deleted in your curl command, such as `/endpoint/?id=${id1},${id2},${id3}`. - If the endpoint contains parameters, specify the parameter values when calling the endpoint. ### Response @@ -349,4 +475,4 @@ To delete an endpoint, perform the following steps: 1. Navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project. 2. In the left pane, click the name of your target Data App to view its endpoints. 3. Click the name of the endpoint you want to delete, and then click **...** > **Delete** in the upper-right corner. -4. Click **Delete** to confirm the deletion. \ No newline at end of file +4. Click **Delete** to confirm the deletion. diff --git a/tidb-cloud/data-service-oas-with-nextjs.md b/tidb-cloud/data-service-oas-with-nextjs.md index 29d3dd2901786..022099ef86ccc 100644 --- a/tidb-cloud/data-service-oas-with-nextjs.md +++ b/tidb-cloud/data-service-oas-with-nextjs.md @@ -11,18 +11,18 @@ This document introduces how to use the OpenAPI Specification of a [Data App](/t Before using OpenAPI Specification with Next.js, make sure that you have the following: -- A TiDB cluster. For more information, see [Create a TiDB Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) or [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). +- A TiDB cluster. For more information, see [Create a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) or [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). - [Node.js](https://nodejs.org/en/download) - [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) - [yarn](https://yarnpkg.com/getting-started/install) -This document uses a TiDB Serverless cluster as an example. +This document uses a TiDB Cloud Serverless cluster as an example. ## Step 1. Prepare data To begin with, create a table `test.repository` in your TiDB cluster and insert some sample data into it. The following example inserts some open source projects developed by PingCAP as data for demonstration purposes. -To execute the SQL statements, you can use [Chat2Query](/tidb-cloud/explore-data-with-chat2query.md) in the [TiDB Cloud console](https://tidbcloud.com). +To execute the SQL statements, you can use [SQL Editor](/tidb-cloud/explore-data-with-chat2query.md) in the [TiDB Cloud console](https://tidbcloud.com). ```sql -- Select the database @@ -182,7 +182,7 @@ You can use the generated client code to develop your Next.js application. > **Note:** > - > If the linked clusters of your Data App are hosted in different regions, you wil see multiple items in the `servers` section of the downloaded OpenAPI Specification file. In this case, you also need to configure the endpoint path in the `config` object as follows: + > If the linked clusters of your Data App are hosted in different regions, you will see multiple items in the `servers` section of the downloaded OpenAPI Specification file. In this case, you also need to configure the endpoint path in the `config` object as follows: > > ```js > const config = new Configuration({ @@ -206,4 +206,4 @@ To preview your application in a local development server, run the following com yarn dev ``` -You can then open in your browser and see the data from the `test.repository` database displayed on the page. +You can then open [http://localhost:3000](http://localhost:3000) in your browser and see the data from the `test.repository` database displayed on the page. diff --git a/tidb-cloud/data-service-overview.md b/tidb-cloud/data-service-overview.md index ccbae3abdeb5f..87eb3dec02312 100644 --- a/tidb-cloud/data-service-overview.md +++ b/tidb-cloud/data-service-overview.md @@ -5,11 +5,13 @@ summary: Learn about Data Service in TiDB Cloud and its scenarios. # TiDB Cloud Data Service (Beta) Overview -TiDB Cloud provides a [Data Service (beta)](https://tidbcloud.com/console/data-service) feature that enables you to access TiDB Cloud data via an HTTPS request using a custom API endpoint. This feature uses a serverless architecture to handle computing resources and elastic scaling, so you can focus on the query logic in endpoints without worrying about infrastructure or maintenance costs. +TiDB Cloud [Data Service (beta)](https://tidbcloud.com/console/data-service) is a fully managed low-code backend-as-a-service solution that simplifies backend application development, empowering developers to rapidly build highly scalable, secure, data-driven applications. + +Data Service enables you to access TiDB Cloud data via an HTTPS request using a custom API endpoint. This feature uses a serverless architecture to handle computing resources and elastic scaling, so you can focus on the query logic in endpoints without worrying about infrastructure or maintenance costs. > **Note:** > -> Data Service is available for [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. To use Data Service in TiDB Dedicated clusters, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). +> Data Service is available for [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters. To use Data Service in TiDB Cloud Dedicated clusters, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). An endpoint in Data Service is a web API that you can customize to execute SQL statements. You can specify parameters for your SQL statements, such as the value used in the `WHERE` clause. When a client calls an endpoint and provides values for the parameters in a request URL, the endpoint executes the corresponding SQL statement with the provided parameters and returns the results as part of the HTTP response. diff --git a/tidb-cloud/data-service-postman-integration.md b/tidb-cloud/data-service-postman-integration.md index 89545832c2d26..4be7d94d8b3bf 100644 --- a/tidb-cloud/data-service-postman-integration.md +++ b/tidb-cloud/data-service-postman-integration.md @@ -72,7 +72,7 @@ To run your Data App in Postman, take the following steps: - For an endpoint with parameters, you need to fill in the parameter values first, and then click **Send**. - For a `GET` or `DELETE` request, fill in the parameter values in the **Query Params** table. - - For a `POST` or `PUT` request, click the **Body** tab, and then fill in the parameter values as a JSON object. If **Batch Operation** is enabled for the endpoint in TiDB Cloud Data Service, fill in the parameter values as an array of JSON objects. + - For a `POST` or `PUT` request, click the **Body** tab, and then fill in the parameter values as a JSON object. If **Batch Operation** is enabled for the endpoint in TiDB Cloud Data Service, fill in the parameter values to the `items` field as an array of JSON objects. 3. Check the response in the lower pane. diff --git a/tidb-cloud/data-service-response-and-status-code.md b/tidb-cloud/data-service-response-and-status-code.md index 7b96fcb34abd4..08b8a0e1ef435 100644 --- a/tidb-cloud/data-service-response-and-status-code.md +++ b/tidb-cloud/data-service-response-and-status-code.md @@ -25,10 +25,10 @@ The response body contains the following fields: - `columns`: _array_. Schema information for the returned fields. - `rows`: _array_. The returned results in `key:value` format. - When **Batch Operation** is enabled for an endpoint and the last SQL statement of the endpoint is an `INSERT`, `UPDATE`, or `DELETE` operation, note the following: + When **Batch Operation** is enabled for an endpoint and the last SQL statement of the endpoint is an `INSERT` or `UPDATE` operation, note the following: - The returned results of the endpoint will also include the `"message"` and `"success"` fields for each row to indicate their response and status. - - If the primary key column of the target table is configured as `auto_increment`, the returned results of the endpoint will also include the `"auto_increment_id"` field for each row. The value of this field is the auto increment ID for an `INSERT` operation and is `null` for other operations such as `UPDATE` and `DELETE`. + - If the primary key column of the target table is configured as `auto_increment`, the returned results of the endpoint will also include the `"auto_increment_id"` field for each row. The value of this field is the auto increment ID for an `INSERT` operation and is `null` for other operations such as `UPDATE`. - `result`: _object_. The execution-related information of the SQL statement, including success/failure status, execution time, number of rows returned, and user configuration. diff --git a/tidb-cloud/delete-tidb-cluster.md b/tidb-cloud/delete-tidb-cluster.md index d8d042d9bbec0..3ff4923c5fda1 100644 --- a/tidb-cloud/delete-tidb-cluster.md +++ b/tidb-cloud/delete-tidb-cluster.md @@ -17,19 +17,26 @@ You can delete a cluster at any time by performing the following steps: > Alternatively, you can also click the name of the target cluster to go to its overview page, and then click **...** in the upper-right corner. 3. Click **Delete** in the drop-down menu. -4. In the cluster deleting window, enter your `//`. +4. In the cluster deleting window, confirm the deletion: - If you want to restore the cluster sometime in the future, make sure that you have a backup of the cluster. Otherwise, you cannot restore it anymore. For more information about how to back up TiDB Dedicated clusters, see [Back Up and Restore TiDB Dedicated Data](/tidb-cloud/backup-and-restore.md). + - If you have at least one manual or automatic backup, you can see the number of backups and the charging policy for backups. Click **Continue** and enter `//`. + - If you do not have any backups, just enter `//`. + + If you want to restore the cluster sometime in the future, make sure that you have a backup of the cluster. Otherwise, you cannot restore it anymore. For more information about how to back up TiDB Cloud Dedicated clusters, see [Back Up and Restore TiDB Cloud Dedicated Data](/tidb-cloud/backup-and-restore.md). > **Note:** > - > [TiDB Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-serverless) only support [in-place restoring from backups](/tidb-cloud/backup-and-restore-serverless.md#restore) and do not support restoring data after the deletion. If you want to delete a TiDB Serverless cluster and restore its data in the future, you can use [Dumpling](https://docs.pingcap.com/tidb/stable/dumpling-overview) to export your data as a backup. + > [TiDB Cloud Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) only support [in-place restoring from backups](/tidb-cloud/backup-and-restore-serverless.md#restore) and do not support restoring data after the deletion. If you want to delete a TiDB Cloud Serverless cluster and restore its data in the future, you can use [Dumpling](https://docs.pingcap.com/tidb/stable/dumpling-overview) to export your data as a backup. + +5. Click **I understand, delete it**. -5. Click **I understand the consequences. Delete this cluster**. + Once a backed up TiDB Cloud Dedicated cluster is deleted, the existing backup files of the cluster are moved to the recycle bin. - Once a backed up TiDB Dedicated cluster is deleted, the existing backup files of the cluster are moved to the recycle bin. + - Automatic backups will expire and be automatically deleted once the retention period ends. The default retention period is 7 days if you don't modify it. + - Manual backups will be kept in the Recycle Bin until manually deleted. -- For backup files from an automatic backup, the recycle bin can retain them for 7 days. -- For backup files from a manual backup, there is no expiration date. + > **Note:** + > + > Please be aware that backups will continue to incur charges until deleted. - If you want to restore a TiDB Dedicated cluster from recycle bin, see [Restore a deleted cluster](/tidb-cloud/backup-and-restore.md#restore-a-deleted-cluster). + If you want to restore a TiDB Cloud Dedicated cluster from recycle bin, see [Restore a deleted cluster](/tidb-cloud/backup-and-restore.md#restore-a-deleted-cluster). diff --git a/tidb-cloud/dev-guide-bi-looker-studio.md b/tidb-cloud/dev-guide-bi-looker-studio.md new file mode 100644 index 0000000000000..4ac654f8e9afb --- /dev/null +++ b/tidb-cloud/dev-guide-bi-looker-studio.md @@ -0,0 +1,137 @@ +--- +title: Connect to TiDB Cloud Serverless with Looker Studio +summary: Learn how to connect to TiDB Cloud Serverless using Looker Studio. +--- + +# Connect to TiDB Cloud Serverless with Looker Studio + +TiDB is a MySQL-compatible database, TiDB Cloud Serverless is a fully managed TiDB offering, and [Looker Studio](https://lookerstudio.google.com/) is a free web-based BI tool that can visualize data from various sources. + +In this tutorial, you can learn how to connect to your TiDB Cloud Serverless cluster with Looker Studio. + +> **Note:** +> +> Most steps in this tutorial work with TiDB Cloud Dedicated as well. However, for TiDB Cloud Dedicated, you need to note the following: +> +> - Import your dataset following [Import data from files to TiDB Cloud](/tidb-cloud/tidb-cloud-migration-overview.md#import-data-from-files-to-tidb-cloud). +> - Get the connection information for your cluster following [Connect to TiDB Cloud Dedicated](/tidb-cloud/connect-via-standard-connection.md). When connecting to TiDB Cloud Dedicated, you need to allow access from `142.251.74.0/23`. For more information about connections from Looker Studio, see [Looker Studio documentation](https://support.google.com/looker-studio/answer/7088031#zippy=%2Cin-this-article). + +## Prerequisites + +To complete this tutorial, you need: + +- A Google account +- A TiDB Cloud Serverless cluster + +**If you don't have a TiDB Cloud Serverless cluster, you can create one as follows:** + +- [Create a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-cloud-cluster) + +## Step 1. Import a dataset + +You can import the S&P 500 dataset provided in the interactive tutorial of TiDB Cloud Serverless. + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and click **?** in the lower-right corner. A **Help** dialog is displayed. + +2. In the dialog, click **Interactive Tutorials**, and then click **S&P 500 Analysis**. + +3. Select your TiDB Cloud Serverless cluster, and then click **Import Dataset** to import the S&P 500 dataset to your cluster. + +4. After the import status changes to **IMPORTED**, click **Exit Tutorial** to close this dialog. + +If you encounter any issues during import, you can cancel this import task as follows: + +1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click the name of your TiDB Cloud Serverless cluster to go to its overview page. +2. In the left navigation pane, click **Import**. +3. Find the import task named **sp500-insight**, click **...** in the **Action** column, and then click **Cancel**. + +## Step 2. Get the connection information for your cluster + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. In the connection dialog, set **Connect With** to `General`, and then click **Generate Password** to create a random password. + + > **Tip:** + > + > If you have created a password before, use the original password or click **Reset Password** to generate a new one. + +4. Download the [CA cert](https://letsencrypt.org/certs/isrgrootx1.pem). + + > **Tip:** + > + > TiDB Cloud Serverless requires a secure TLS connection between the client and the cluster, so you need this CA cert for connection settings in Looker Studio. + +## Step 3. Connect to your TiDB cluster with Looker Studio + +1. Log into [Looker Studio](https://lookerstudio.google.com/), and then click **Create** > **Report** in the left navigation pane. + +2. On the displayed page, search and select the **MySQL** connector, and then click **AUTHORIZE**. + +3. In the **BASIC** setting pane, configure the connection parameters. + + - **Host Name or IP**: enter the `HOST` parameter from the TiDB Cloud Serverless connection dialog. + - **Port(Optional)**: enter the `PORT` parameter from the TiDB Cloud Serverless connection dialog. + - **Database**: enter the database you want to connect to. For this tutorial, enter `sp500insight`. + - **Username**: enter the `USERNAME` parameter from the TiDB Cloud Serverless connection dialog. + - **Password**: enter the `PASSWORD` parameter from the TiDB Cloud Serverless connection dialog. + - **Enable SSL**: select this option, and then click the upload icon to the right of **MySQL SSL Client Configuration Files** to upload the CA file downloaded from [Step 2](#step-2-get-the-connection-information-for-your-cluster). + + ![Looker Studio: configure connection settings for TiDB Cloud Serverless](/media/tidb-cloud/looker-studio-configure-connection.png) + +4. Click **AUTHENTICATE**. + +If the authentication succeeds, you can see tables in the database. + +## Step 4. Create a simple chart + +Now, you can use the TiDB cluster as a data source and create a simple chart with data. + +1. In the right pane, click **CUSTOM QUERY**. + + ![Looker Studio: custom query](/media/tidb-cloud/looker-studio-custom-query.png) + +2. Copy the following code to the **Enter Custom Query** area, and then click **Add** in the lower-right corner. + + ```sql + SELECT sector, + COUNT(*) AS companies, + ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC ) AS companies_ranking, + SUM(market_cap) AS total_market_cap, + ROW_NUMBER() OVER (ORDER BY SUM(market_cap) DESC ) AS total_market_cap_ranking, + SUM(revenue_growth * weight) / SUM(weight) AS avg_revenue_growth, + ROW_NUMBER() OVER (ORDER BY SUM(revenue_growth * weight) / SUM(weight) DESC ) AS avg_revenue_growth_ranking + FROM companies + LEFT JOIN index_compositions ic ON companies.stock_symbol = ic.stock_symbol + GROUP BY sector + ORDER BY 5 ASC; + ``` + + If you see the **You are about to add data to this report** dialog, click **ADD TO REPORT**. Then, a table is displayed in the report. + +3. In the toolbar of the report, click **Add a chart**, and then select `Combo chart` in the `Line` category. + +4. In the **Chart** settings pane on the right, configure the following parameters: + + - In the **SETUP** Tab: + - **Dimension**: `sector`. + - **Metric**: `companies` and `total_market_cap`. + - In the **STYLE** Tab: + - Series #1: select the `Line` option and the `Right` axis. + - Series #2: select the `Bars` option and the `Left` axis. + - Leave other fields as defaults. + +Then, you can see a combo chart similar as follows: + +![Looker Studio: A simple Combo chart](/media/tidb-cloud/looker-studio-simple-chart.png) + +## Next steps + +- Learn more usage of Looker Studio from [Looker Studio Help](https://support.google.com/looker-studio). +- Learn the best practices for TiDB application development with the chapters in the [Developer guide](/develop/dev-guide-overview.md), such as [Insert data](/develop/dev-guide-insert-data.md), [Update data](/develop/dev-guide-update-data.md), [Delete data](/develop/dev-guide-delete-data.md), [Single table reading](/develop/dev-guide-get-data-from-single-table.md), [Transactions](/develop/dev-guide-transaction-overview.md), and [SQL performance optimization](/develop/dev-guide-optimize-sql-overview.md). +- Learn through the professional [TiDB developer courses](https://www.pingcap.com/education/) and earn [TiDB certifications](https://www.pingcap.com/education/certification/) after passing the exam. + +## Need help? + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). diff --git a/tidb-cloud/dev-guide-wordpress.md b/tidb-cloud/dev-guide-wordpress.md new file mode 100644 index 0000000000000..b1d837040621c --- /dev/null +++ b/tidb-cloud/dev-guide-wordpress.md @@ -0,0 +1,108 @@ +--- +title: Connect to TiDB Cloud Serverless with WordPress +summary: Learn how to use TiDB Cloud Serverless to run WordPress. This tutorial gives step-by-step guidance to run WordPress + TiDB Cloud Serverless in a few minutes. +--- + +# Connect to TiDB Cloud Serverless with WordPress + +TiDB is a MySQL-compatible database, TiDB Cloud Serverless is a fully managed TiDB offering, and [WordPress](https://github.com/WordPress) is a free, open-source content management system (CMS) that lets users create and manage websites. WordPress is written in PHP and uses a MySQL database. + +In this tutorial, you can learn how to use TiDB Cloud Serverless to run WordPress for free. + +> **Note:** +> +> In addition to TiDB Cloud Serverless, this tutorial works with TiDB Cloud Dedicated and TiDB Self-Managed clusters as well. However, it is highly recommended to run WordPress with TiDB Cloud Serverless for cost efficiency. + +## Prerequisites + +To complete this tutorial, you need: + +- A TiDB Cloud Serverless cluster. Follow [creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster if you don't have one. + +## Run WordPress with TiDB Cloud Serverless + +This section demonstrates how to run WordPress with TiDB Cloud Serverless. + +### Step 1: Clone the WordPress sample repository + +Run the following commands in your terminal window to clone the sample code repository: + +```shell +git clone https://github.com/Icemap/wordpress-tidb-docker.git +cd wordpress-tidb-docker +``` + +### Step 2: Install dependencies + +1. The sample repository requires [Docker](https://www.docker.com/) and [Docker Compose](https://docs.docker.com/compose/) to start WordPress. If you have them installed, you can skip this step. It is highly recommended to run your WordPress in a Linux environment (such as Ubuntu). Run the following command to install Docker and Docker Compose: + + ```shell + sudo sh install.sh + ``` + +2. The sample repository includes the [TiDB Compatibility Plugin](https://github.com/pingcap/wordpress-tidb-plugin) as a submodule. Run the following command to update the submodule: + + ```shell + git submodule update --init --recursive + ``` + +### Step 3: Configure connection information + +Configure the WordPress database connection to TiDB Cloud Serverless. + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Ensure the configurations in the connection dialog match your operating environment. + + - **Connection Type** is set to `Public`. + - **Connect With** is set to `WordPress`. + - **Operating System** is set to `Debian/Ubuntu/Arch`. + - **Database** is set to the database you want to use—for example, `test`. + +4. Click **Generate Password** to create a random password. + + > **Tip:** + > + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. + +5. Run the following command to copy `.env.example` and rename it to `.env`: + + ```shell + cp .env.example .env + ``` + +6. Copy and paste the corresponding connection string into the `.env` file. The example result is as follows: + + ```dotenv + TIDB_HOST='{HOST}' # e.g. gateway01.ap-northeast-1.prod.aws.tidbcloud.com + TIDB_PORT='4000' + TIDB_USER='{USERNAME}' # e.g. xxxxxx.root + TIDB_PASSWORD='{PASSWORD}' + TIDB_DB_NAME='test' + ``` + + Be sure to replace the placeholders `{}` with the connection parameters obtained from the connection dialog. By default, your TiDB Cloud Serverless comes with a `test` database. If you have already created another database in your TiDB Cloud Serverless cluster, you can replace `test` with your database name. + +7. Save the `.env` file. + +### Step 4: Start WordPress with TiDB Cloud Serverless + +1. Execute the following command to run WordPress as a Docker container: + + ```shell + docker compose up -d + ``` + +2. Set up your WordPress site by visiting [localhost](http://localhost/) if you start the container on your local machine or `http://` if the WordPress is running on a remote machine. + +### Step 5: Confirm the database connection + +1. Close the connection dialog for your cluster on the TiDB Cloud console, and open the **SQL Editor** page. +2. Under the **Schemas** tab on the left, click the database you connected to Wordpress. +3. Confirm that you now see the Wordpress tables (such as `wp_posts` and `wp_comments`) in the list of tables for that database. + +## Need help? + +Ask questions on [TiDB Community](https://ask.pingcap.com/), or [create a support ticket](https://support.pingcap.com/). diff --git a/tidb-cloud/explore-data-with-chat2query.md b/tidb-cloud/explore-data-with-chat2query.md index 9090e342ad2e1..13d8cd08f9e24 100644 --- a/tidb-cloud/explore-data-with-chat2query.md +++ b/tidb-cloud/explore-data-with-chat2query.md @@ -1,35 +1,29 @@ --- -title: Explore Your Data with AI-Powered Chat2Query (beta) -summary: Learn how to use Chat2Query, an AI-powered SQL editor in the TiDB Cloud console, to maximize your data value. +title: Explore Your Data with AI-Assisted SQL Editor +summary: Learn how to use AI-assisted SQL Editor in the TiDB Cloud console, to maximize your data value. --- -# Explore Your Data with AI-Powered Chat2Query (beta) +# Explore Your Data with AI-Assisted SQL Editor -TiDB Cloud is powered by AI. You can use Chat2Query (beta), an AI-powered SQL editor in the [TiDB Cloud console](https://tidbcloud.com/), to maximize your data value. +You can use the built-in AI-assisted SQL Editor in the [TiDB Cloud console](https://tidbcloud.com/) to maximize your data value. -In Chat2Query, you can either simply type `--` followed by your instructions to let AI generate SQL queries automatically or write SQL queries manually, and then run SQL queries against databases without a terminal. You can find the query results in tables intuitively and check the query logs easily. - -> **Note:** -> -> Chat2Query is supported for TiDB clusters that are v6.5.0 or later and are hosted on AWS. -> -> - For [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters, Chat2Query is available by default. -> - For [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters, Chat2Query is only available upon request. To use Chat2Query on TiDB Dedicated clusters, contact [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md). +In SQL Editor, you can either write SQL queries manually or simply press + I on macOS (or Control + I on Windows or Linux) to instruct [Chat2Query (beta)](/tidb-cloud/tidb-cloud-glossary.md#chat2query) to generate SQL queries automatically. This enables you to run SQL queries against databases without a local SQL client. You can intuitively view the query results in tables or charts and easily check the query logs. ## Use cases -The recommended use cases of Chat2Query are as follows: +The recommended use cases of SQL Editor are as follows: -- Use the AI capacity of Chat2Query to help you generate complex SQL queries instantly. -- Test out the MySQL compatibility of TiDB quickly. -- Explore TiDB SQL features easily. +- Utilize the AI capabilities of Chat2Query to generate, debug, or rewrite complex SQL queries instantly. +- Quickly test the MySQL compatibility of TiDB. +- Easily explore the SQL features in TiDB using your own datasets. -## Limitation +## Limitations -- SQL queries generated by the AI are not 100% accurate and might still need your further tweak. -- The [Chat2Query API](/tidb-cloud/use-chat2query-api.md) is available for [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. To use the Chat2Query API on [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters, contact [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md). +- SQL queries generated by the AI might not be 100% accurate, and you might need to refine them. +- SQL Editor is only supported for TiDB clusters that are v6.5.0 or later and hosted on AWS. +- SQL Editor is available by default for [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters. To use SQL Editor and Chat2Query on [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters, contact [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md). -## Access Chat2Query +## Access SQL Editor 1. Go to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. @@ -37,19 +31,19 @@ The recommended use cases of Chat2Query are as follows: > > If you have multiple projects, you can click in the lower-left corner and switch to another project. -2. Click your cluster name, and then click **Chat2Query** in the left navigation pane. +2. Click your cluster name, and then click **SQL Editor** in the left navigation pane. > **Note:** > - > In the following cases, the **Chat2Query** entry is displayed in grey and not clickable. + > In the following cases, the **SQL Editor** entry is displayed in gray and not clickable. > - > - Your TiDB Dedicated cluster is earlier than v6.5.0. To use Chat2Query, you need to contract [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md) to upgrade your clusters. - > - Your TiDB Dedicated cluster is just created and the running environment for Chat2Query is still in preparation. In this case, wait for a few minutes and Chat2Query will be available. - > - Your TiDB Dedicated cluster is [paused](/tidb-cloud/pause-or-resume-tidb-cluster.md). + > - Your TiDB Cloud Dedicated cluster is earlier than v6.5.0. To use SQL Editor, you need to contract [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md) to upgrade your clusters. + > - Your TiDB Cloud Dedicated cluster is just created and the running environment for SQL Editor is still in preparation. In this case, wait for a few minutes and Chat2Query will be available. + > - Your TiDB Cloud Dedicated cluster is [paused](/tidb-cloud/pause-or-resume-tidb-cluster.md). ## Enable or disable AI to generate SQL queries -PingCAP takes the privacy and security of users' data as a top priority. The AI capacity of Chat2Query only needs to access database schemas to generate SQL queries, not your data itself. For more information, see [Chat2Query Privacy FAQ](https://www.pingcap.com/privacy-policy/privacy-chat2query). +PingCAP takes the privacy and security of users' data as a top priority. The AI capacity of Chat2Query in SQL Editor only needs to access database schemas to generate SQL queries, not your data itself. For more information, see [Chat2Query Privacy FAQ](https://www.pingcap.com/privacy-policy/privacy-chat2query). When you access Chat2Query for the first time, you will be prompted with a dialog about whether to allow PingCAP and OpenAI to use your code snippets to research and improve the services. @@ -63,16 +57,36 @@ After the first-time access, you can still change the AI setting as follows: ## Write and run SQL queries -In Chat2Query, you can write and run SQL queries using your own dataset. +In SQL Editor, you can write and run SQL queries using your own dataset. 1. Write SQL queries. - - If AI is enabled, simply type `--` followed by your instructions to let AI generate SQL queries automatically or write SQL queries manually. + +
    - For a SQL query generated by AI, you can accept it by pressing Tab and then further edit it if needed, or reject it by pressing Esc. + For macOS: + + - If AI is enabled, simply press **⌘ + I** followed by your instructions and press **Enter** to let AI generate SQL queries automatically. + + For a SQL query generated by Chat2Query, click **Accept** to accept the query and continue editing. If the query does not meet your requirements, click **Discard** to reject it. Alternatively, click **Regenerate** to request a new query from Chat2Query. - If AI is disabled, write SQL queries manually. +
    + +
    + + For Windows or Linux: + + - If AI is enabled, simply press **Ctrl + I** followed by your instructions and press **Enter** to let AI generate SQL queries automatically. + + For a SQL query generated by Chat2Query, click **Accept** to accept the query and continue editing. If the query does not meet your requirements, click **Discard** to reject it. Alternatively, click **Regenerate** to request a new query from Chat2Query. + + - If AI is disabled, write SQL queries manually. + +
    +
    + 2. Run SQL queries. @@ -103,17 +117,58 @@ In Chat2Query, you can write and run SQL queries using your own dataset. After running the queries, you can see the query logs and results immediately at the bottom of the page. +> **Note:** +> +> The returned result has a size limit of 8 MiB. + +## Rewrite SQL queries using Chat2Query + +In SQL Editor, you can use Chat2Query to rewrite existing SQL queries to optimize performance, fix errors, or meet other specific requirements. + +1. Select the lines of SQL queries you want to rewrite with your cursor. + +2. Invoke Chat2Query for rewriting by using the keyboard shortcut for your operating system: + + - + I on macOS + - Control + I on Windows or Linux + + Press **Enter** after providing your instructions to let AI handle the rewrite. + +3. After invoking Chat2Query, you can see the suggested rewrite and the following options: + + - **Accept**: click this to accept the suggested rewrite and continue editing. + - **Discard**: click this if the suggested rewrite does not meet your expectations. + - **Regenerate**: click this to request another rewrite from Chat2Query based on your feedback or additional instructions. + +> **Note:** +> +> Chat2Query uses AI algorithms to suggest optimizations and corrections. It is recommended that you review these suggestions carefully before finalizing the queries. + ## Manage SQL files -In Chat2Query, you can save your SQL queries in different SQL files and manage SQL files as follows: +In SQL Editor, you can save your SQL queries in different SQL files and manage SQL files as follows: - To add a SQL file, click **+** on the **SQL Files** tab. - To rename a SQL file, move your cursor on the filename, click **...** next to the filename, and then select **Rename**. - To delete a SQL file, move your cursor on the filename, click **...** next to the filename, and then select **Delete**. Note that when there is only one SQL file on the **SQL Files** tab, you cannot delete it. +## Access Chat2Query via API + +In addition to accessing Chat2Query via UI, you can also access Chat2Query via API. To do so, you need to create a Chat2Query Data App first. + +In Chat2Query, you can access or create a Chat2Query Data App as follows: + +1. Click **...** in the upper-right corner, and then click **Access Chat2Query via API**. +2. In the displayed dialog, do one of the following: + + - To create a new Chat2Query Data App, click **New Chat2Query Data App**. + - To access an existing Chat2Query Data App, click the name of target Data App. + +For more information, see [Get started with Chat2Query API](/tidb-cloud/use-chat2query-api.md). + ## Generate an endpoint from a SQL file -For TiDB clusters, TiDB Cloud provides a [Data Service (beta)](/tidb-cloud/data-service-overview.md) feature that enables you to access TiDB Cloud data via an HTTPS request using a custom API endpoint. In Chat2Query, you can generate an endpoint of Data Service (beta) from a SQL file by taking the following steps: +For TiDB clusters, TiDB Cloud provides a [Data Service (beta)](/tidb-cloud/data-service-overview.md) feature that enables you to access TiDB Cloud data via an HTTPS request using a custom API endpoint. In SQL Editor, you can generate an endpoint of Data Service (beta) from a SQL file by taking the following steps: 1. Move your cursor on the filename, click **...** next to the filename, and then select **Generate endpoint**. 2. In the **Generate endpoint** dialog box, select the Data App you want to generate the endpoint for and enter the endpoint name. @@ -121,16 +176,15 @@ For TiDB clusters, TiDB Cloud provides a [Data Service (beta)](/tidb-cloud/data- For more information, see [Manage an endpoint](/tidb-cloud/data-service-manage-endpoint.md). -## Manage Chat2Query settings +## Manage SQL Editor settings -In Chat2Query, you can change the following settings: +In SQL Editor, you can change the following settings: - The maximum number of rows in query results - Whether to show system database schemas on the **Schemas** tab -- Whether to enable [Chat2Query API](/tidb-cloud/use-chat2query-api.md) To change the settings, take the following steps: -1. In the upper-right corner of Chat2Query, click **...** and select **Settings**. +1. In the upper-right corner of **SQL Editor**, click **...** and select **Settings**. 2. Change the settings according to your need. 3. Click **Save**. diff --git a/tidb-cloud/get-started-with-cli.md b/tidb-cloud/get-started-with-cli.md index c5d86ed720727..db21975b117e2 100644 --- a/tidb-cloud/get-started-with-cli.md +++ b/tidb-cloud/get-started-with-cli.md @@ -8,12 +8,16 @@ summary: Learn how to manage TiDB Cloud resources through the TiDB Cloud CLI. TiDB Cloud provides a command-line interface (CLI) [`ticloud`](https://github.com/tidbcloud/tidbcloud-cli) for you to interact with TiDB Cloud from your terminal with a few lines of commands. For example, you can easily perform the following operations using `ticloud`: - Create, delete, and list your clusters. -- Import data from Amazon S3 or local files to your clusters. +- Import data to your clusters. +- Export data from your clusters. + +> **Note:** +> +> TiDB Cloud CLI is in beta. ## Before you begin - Have a TiDB Cloud account. If you do not have one, [sign up for a free trial](https://tidbcloud.com/free-trial). -- [Create a TiDB Cloud API Key](https://docs.pingcap.com/tidbcloud/api/v1beta#section/Authentication/API-Key-Management). ## Installation @@ -81,6 +85,44 @@ Install the MySQL command-line client if you do not have it. You can refer to th
    +## Quick start + +[TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) is the best way to get started with TiDB Cloud. In this section, you will learn how to create a TiDB Cloud Serverless cluster with TiDB Cloud CLI. + +### Create a user profile or log into TiDB Cloud + +Before creating a cluster with TiDB Cloud CLI, you need to either create a user profile or log into TiDB Cloud. + +- Create a user profile with your [TiDB Cloud API key](https://docs.pingcap.com/tidbcloud/api/v1beta#section/Authentication/API-Key-Management): + + ```shell + ticloud config create + ``` + + > **Warning:** + > + > The profile name **MUST NOT** contain `.`. + +- Log into TiDB Cloud with authentication: + + ```shell + ticloud auth login + ``` + + After successful login, an OAuth token will be assigned to the current profile. If no profiles exist, the token will be assigned to a profile named `default`. + +> **Note:** +> +> In the preceding two methods, the TiDB Cloud API key takes precedence over the OAuth token. If both are available, the API key will be used. + +### Create a TiDB Cloud Serverless cluster + +To create a TiDB Cloud Serverless cluster, enter the following command, and then follow the CLI prompts to provide the required information: + +```shell +ticloud serverless create +``` + ## Use the TiDB Cloud CLI View all commands available: @@ -114,7 +156,7 @@ tiup cloud --help Run commands with `tiup cloud `. For example: ```shell -tiup cloud cluster create +tiup cloud serverless create ``` Update to the latest version by TiUP: @@ -123,40 +165,6 @@ Update to the latest version by TiUP: tiup update cloud ``` -## Quick start - -[TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) is the best way to get started with TiDB Cloud. In this section, you will learn how to create a TiDB Serverless cluster with TiDB Cloud CLI. - -### Create a user profile - -Before creating a cluster, you need to create a user profile with your TiDB Cloud API Key: - -```shell -ticloud config create -``` - -> **Warning:** -> -> The profile name **MUST NOT** contain `.`. - -### Create a TiDB Serverless cluster - -To create a TiDB Serverless cluster, enter the following command, and then follow the CLI prompts to provide the required information and set the password: - -```shell -ticloud cluster create -``` - -### Connect to the cluster - -After the cluster is created, you can connect to the cluster: - -```shell -ticloud connect -``` - -When you are prompted about whether to use the default user, choose `Y` and enter the password that you set when creating the cluster. - ## What's next Check out [CLI reference](/tidb-cloud/cli-reference.md) to explore more features of TiDB Cloud CLI. diff --git a/tidb-cloud/high-availability-with-multi-az.md b/tidb-cloud/high-availability-with-multi-az.md index eb4b0ecb3987b..1741634147c42 100644 --- a/tidb-cloud/high-availability-with-multi-az.md +++ b/tidb-cloud/high-availability-with-multi-az.md @@ -7,7 +7,7 @@ summary: TiDB Cloud supports high availability with Multi-AZ deployments. TiDB uses the Raft consensus algorithm to ensure that data is highly available and safely replicated throughout storage in Raft Groups. Data is redundantly copied between storage nodes and placed in different availability zones to protect against machine or data center failures. With automatic failover, TiDB ensures that your service is always on. -TiDB Cloud clusters consist of three major components: TiDB node, TiKV node, and TiFlash node. The highly availability implementation of each component for TiDB Dedicated is as follows: +TiDB Cloud clusters consist of three major components: TiDB node, TiKV node, and TiFlash node. The highly availability implementation of each component for TiDB Cloud Dedicated is as follows: * **TiDB node** diff --git a/tidb-cloud/import-csv-files.md b/tidb-cloud/import-csv-files.md index d668e1c27b8fd..ef49a2d66b2e4 100644 --- a/tidb-cloud/import-csv-files.md +++ b/tidb-cloud/import-csv-files.md @@ -8,10 +8,11 @@ aliases: ['/tidbcloud/migrate-from-amazon-s3-or-gcs','/tidbcloud/migrate-from-au This document describes how to import CSV files from Amazon Simple Storage Service (Amazon S3) or Google Cloud Storage (GCS) into TiDB Cloud. -> **Note:** -> -> - To ensure data consistency, TiDB Cloud allows to import CSV files into empty tables only. To import data into an existing table that already contains data, you can use TiDB Cloud to import the data into a temporary empty table by following this document, and then use the `INSERT SELECT` statement to copy the data to the target existing table. -> - If there is a changefeed in a TiDB Dedicated cluster, you cannot import data to the cluster (the **Import Data** button will be disabled), because the current import data feature uses the [physical import mode](https://docs.pingcap.com/tidb/stable/tidb-lightning-physical-import-mode). In this mode, the imported data does not generate change logs, so the changefeed cannot detect the imported data. +## Limitations + +- To ensure data consistency, TiDB Cloud allows to import CSV files into empty tables only. To import data into an existing table that already contains data, you can use TiDB Cloud to import the data into a temporary empty table by following this document, and then use the `INSERT SELECT` statement to copy the data to the target existing table. + +- If a TiDB Cloud Dedicated cluster has a [changefeed](/tidb-cloud/changefeed-overview.md) or has [Point-in-time Restore](/tidb-cloud/backup-and-restore.md#turn-on-point-in-time-restore) enabled, you cannot import data to the cluster (the **Import Data** button will be disabled), because the current import data feature uses the [physical import mode](https://docs.pingcap.com/tidb/stable/tidb-lightning-physical-import-mode). In this mode, the imported data does not generate change logs, so the changefeed and Point-in-time Restore cannot detect the imported data. ## Step 1. Prepare the CSV files @@ -29,6 +30,7 @@ This document describes how to import CSV files from Amazon Simple Storage Servi > > - You only need to compress the data files, not the database or table schema files. > - To achieve better performance, it is recommended to limit the size of each compressed file to 100 MiB. + > - The Snappy compressed file must be in the [official Snappy format](https://github.com/google/snappy). Other variants of Snappy compression are not supported. > - For uncompressed files, if you cannot update the CSV filenames according to the preceding rules in some cases (for example, the CSV file links are also used by your other programs), you can keep the filenames unchanged and use the **Mapping Settings** in [Step 4](#step-4-import-csv-files-to-tidb-cloud) to import your source data to a single target table. ## Step 2. Create the target table schemas @@ -53,6 +55,10 @@ Because CSV files do not contain schema information, before importing data from CREATE DATABASE mydb; ``` + > **Note:** + > + > Each `${db_name}-schema-create.sql` file must only contain a single DDL statement for creating the database. Otherwise it might fail to create the database. + 2. Create table schema files for your source data. If you do not include the table schema files in the Amazon S3 or GCS directory where the CSV files are located, TiDB Cloud will not create the corresponding tables for you when you import the data. @@ -72,7 +78,7 @@ Because CSV files do not contain schema information, before importing data from > **Note:** > - > Each `${db_name}.${table_name}-schema.sql` file should only contain a single DDL statement. If the file contains multiple DDL statements, only the first one takes effect. + > Each `${db_name}.${table_name}-schema.sql` file must only contain a single DDL statement for creating the table. Otherwise it might fail to create the table. ## Step 3. Configure cross-account access @@ -88,6 +94,9 @@ To allow TiDB Cloud to access the CSV files in the Amazon S3 or GCS bucket, do o To import the CSV files to TiDB Cloud, take the following steps: + +
    + 1. Open the **Import** page for your target cluster. 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. @@ -98,67 +107,110 @@ To import the CSV files to TiDB Cloud, take the following steps: 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. -2. On the **Import** page: - - For a TiDB Dedicated cluster, click **Import Data** in the upper-right corner. - - For a TiDB Serverless cluster, click the **import data from S3** link above the upload area. +2. Select **Import data from S3**. -3. Provide the following information for the source CSV files: + If this is your first time importing data into this cluster, select **Import From Amazon S3**. - - **Location**: select **Amazon S3**. - - **Data Format**: select **CSV**. If you need to edit the CSV configurations, click **Edit CSV configuration** to update the CSV-specific configurations. For more information, see [CSV Configurations for Importing Data](/tidb-cloud/csv-config-for-import-data.md). +3. On the **Import Data from Amazon S3** page, provide the following information for the source CSV files: - > **Note:** - > - > For the configurations of separator and delimiter, you can use both alphanumeric characters and certain special characters. The supported special characters include `\t`, `\b`, `\n`, `\r`, `\f`, and `\u0001`. - - - **Bucket URI**: select the bucket URI where your CSV files are located. Note that you must include `/` at the end of the URI, for example, `s3://sampledate/ingest/`. - - **Bucket Access** (This field is visible only for AWS S3): you can use either an AWS access key or an AWS Role ARN to access your bucket. For more information, see [Configure Amazon S3 access](/tidb-cloud/config-s3-and-gcs-access.md#configure-amazon-s3-access). - - **AWS Access Keys**: enter the AWS access key ID and AWS secret access key. + - **Import File Count**: select **One file** or **Multiple files** as needed. + - **Included Schema Files**: this field is only visible when importing multiple files. If the source folder contains the target table schemas, select **Yes**. Otherwise, select **No**. + - **Data Format**: select **CSV**. + - **File URI** or **Folder URI**: + - When importing one file, enter the source file URI and name in the following format `s3://[bucket_name]/[data_source_folder]/[file_name].csv`. For example, `s3://sampledata/ingest/TableName.01.csv`. + - When importing multiple files, enter the source file URI and name in the following format `s3://[bucket_name]/[data_source_folder]/`. For example, `s3://sampledata/ingest/`. + - **Bucket Access**: you can use either an AWS Role ARN or an AWS access key to access your bucket. For more information, see [Configure Amazon S3 access](/tidb-cloud/config-s3-and-gcs-access.md#configure-amazon-s3-access). - **AWS Role ARN**: enter the AWS Role ARN value. + - **AWS Access Key**: enter the AWS access key ID and AWS secret access key. + +4. Click **Connect**. -4. You can choose to **Import into Pre-created Tables**, or **Import Schema and Data from S3**. +5. In the **Destination** section, select the target database and table. - - **Import into Pre-created Tables** allows you to create tables in TiDB in advance and select the tables that you want to import data into. In this case, you can choose up to 1000 tables to import. To create tables, click on **Chat2Qury** in the left navigation pane. For more information about how to use Chat2Qury, see [Explore Your Data with AI-Powered Chat2Query](/tidb-cloud/explore-data-with-chat2query.md). - - **Import Schema and Data from S3** allows you to import SQL scripts that creates a table along with its corresponding data stored in S3 directly into TiDB. + When importing multiple files, you can use **Advanced Settings** > **Mapping Settings** to define a custom mapping rule for each target table and its corresponding CSV file. After that, the data source files will be re-scanned using the provided custom mapping rule. -5. If the source files do not meet the naming conventions, you can define a custom mapping rule for each target table and its corresponding CSV file. After that, the data source files will be re-scanned using the provided custom mapping rule. To modify the mapping, go to **Advanced Settings** and then click **Mapping Settings**. Note that **Mapping Settings** is available only when you choose **Import into Pre-created Tables**. + When you enter the source file URI and name in **Source File URIs and Names**, make sure it is in the following format `s3://[bucket_name]/[data_source_folder]/[file_name].csv`. For example, `s3://sampledata/ingest/TableName.01.csv`. - - **Target Database**: enter the name of the target database you select. + You can also use wildcards to match the source files. For example: - - **Target Tables**: enter the name of the target table you select. Note that this field only accepts one specific table name, so wildcards are not supported. + - `s3://[bucket_name]/[data_source_folder]/my-data?.csv`: all CSV files starting with `my-data` followed by one character (such as `my-data1.csv` and `my-data2.csv`) in that folder will be imported into the same target table. - - **Source file URIs and names**: enter the source file URI and name in the following format `s3://[bucket_name]/[data_source_folder]/[file_name].csv`. For example, `s3://sampledate/ingest/TableName.01.csv`. You can also use wildcards to match the source files. For more information, see [Mapping Settings](#mapping-settings). + - `s3://[bucket_name]/[data_source_folder]/my-data*.csv`: all CSV files in the folder starting with `my-data` will be imported into the same target table. + + Note that only `?` and `*` are supported. + + > **Note:** + > + > The URI must contain the data source folder. 6. Click **Start Import**. 7. When the import progress shows **Completed**, check the imported tables. -When you run an import task, if any unsupported or invalid conversions are detected, TiDB Cloud terminates the import job automatically and reports an importing error. +
    -If you get an importing error, do the following: +
    -1. Drop the partially imported table. -2. Check the table schema file. If there are any errors, correct the table schema file. -3. Check the data types in the CSV files. -4. Try the import task again. +1. Open the **Import** page for your target cluster. -## Mapping Settings + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. -If the source files do not meet the naming conventions, you can define a custom mapping rule for each target table and its corresponding CSV file. After that, the data source files will be re-scanned using the provided custom mapping rule. To modify the mapping, go to **Advanced Settings** and then click **Mapping Settings**. Note that **Mapping Settings** is available only when you choose **Import into Pre-created Tables**. + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. -When you enter the source file URI and name in **Source file URIs and names**, make sure it is in the following format `s3://[bucket_name]/[data_source_folder]/[file_name].csv`. For example, `s3://sampledate/ingest/TableName.01.csv`. + 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. -You can also use wildcards to match the source files. For example: +2. Click **Import Data** in the upper-right corner. -- `s3://[bucket_name]/[data_source_folder]/my-data?.csv`: all CSV files starting with `my-data` followed by one character (such as `my-data1.csv` and `my-data2.csv`) in that folder will be imported into the same target table. + If this is your first time importing data into this cluster, select **Import From GCS**. -- `s3://[bucket_name]/[data_source_folder]/my-data*.csv`: all CSV files in the folder starting with `my-data` will be imported into the same target table. +3. On the **Import Data from GCS** page, provide the following information for the source CSV files: -Note that only `?` and `*` are supported. + - **Import File Count**: select **One file** or **Multiple files** as needed. + - **Included Schema Files**: this field is only visible when importing multiple files. If the source folder contains the target table schemas, select **Yes**. Otherwise, select **No**. + - **Data Format**: select **CSV**. + - **File URI** or **Folder URI**: + - When importing one file, enter the source file URI and name in the following format `gs://[bucket_name]/[data_source_folder]/[file_name].csv`. For example, `gs://sampledata/ingest/TableName.01.csv`. + - When importing multiple files, enter the source file URI and name in the following format `gs://[bucket_name]/[data_source_folder]/`. For example, `gs://sampledata/ingest/`. + - **Bucket Access**: you can use a GCS IAM Role to access your bucket. For more information, see [Configure GCS access](/tidb-cloud/config-s3-and-gcs-access.md#configure-gcs-access). -> **Note:** -> -> The URI must contain the data source folder. +4. Click **Connect**. + +5. In the **Destination** section, select the target database and table. + + When importing multiple files, you can use **Advanced Settings** > **Mapping Settings** to define a custom mapping rule for each target table and its corresponding CSV file. After that, the data source files will be re-scanned using the provided custom mapping rule. + + When you enter the source file URI and name in **Source File URIs and Names**, make sure it is in the following format `gs://[bucket_name]/[data_source_folder]/[file_name].csv`. For example, `gs://sampledata/ingest/TableName.01.csv`. + + You can also use wildcards to match the source files. For example: + + - `gs://[bucket_name]/[data_source_folder]/my-data?.csv`: all CSV files starting with `my-data` followed by one character (such as `my-data1.csv` and `my-data2.csv`) in that folder will be imported into the same target table. + + - `gs://[bucket_name]/[data_source_folder]/my-data*.csv`: all CSV files in the folder starting with `my-data` will be imported into the same target table. + + Note that only `?` and `*` are supported. + + > **Note:** + > + > The URI must contain the data source folder. + +6. Click **Start Import**. + +7. When the import progress shows **Completed**, check the imported tables. + +
    + +
    + +When you run an import task, if any unsupported or invalid conversions are detected, TiDB Cloud terminates the import job automatically and reports an importing error. + +If you get an importing error, do the following: + +1. Drop the partially imported table. +2. Check the table schema file. If there are any errors, correct the table schema file. +3. Check the data types in the CSV files. +4. Try the import task again. ## Troubleshooting diff --git a/tidb-cloud/import-parquet-files.md b/tidb-cloud/import-parquet-files.md index 0e324093a3d22..db9bd94a49eb4 100644 --- a/tidb-cloud/import-parquet-files.md +++ b/tidb-cloud/import-parquet-files.md @@ -10,8 +10,9 @@ You can import both uncompressed and Snappy compressed [Apache Parquet](https:// > **Note:** > > - TiDB Cloud only supports importing Parquet files into empty tables. To import data into an existing table that already contains data, you can use TiDB Cloud to import the data into a temporary empty table by following this document, and then use the `INSERT SELECT` statement to copy the data to the target existing table. -> - If there is a changefeed in a TiDB Dedicated cluster, you cannot import data to the cluster (the **Import Data** button will be disabled), because the current import data feature uses the [physical import mode](https://docs.pingcap.com/tidb/stable/tidb-lightning-physical-import-mode). In this mode, the imported data does not generate change logs, so the changefeed cannot detect the imported data. -> - Only TiDB Dedicated clusters support importing Parquet files from GCS. +> - If there is a changefeed in a TiDB Cloud Dedicated cluster, you cannot import data to the cluster (the **Import Data** button will be disabled), because the current import data feature uses the [physical import mode](https://docs.pingcap.com/tidb/stable/tidb-lightning-physical-import-mode). In this mode, the imported data does not generate change logs, so the changefeed cannot detect the imported data. +> - Only TiDB Cloud Dedicated clusters support importing Parquet files from GCS. +> - The Snappy compressed file must be in the [official Snappy format](https://github.com/google/snappy). Other variants of Snappy compression are not supported. ## Step 1. Prepare the Parquet files @@ -95,6 +96,9 @@ To allow TiDB Cloud to access the Parquet files in the Amazon S3 or GCS bucket, To import the Parquet files to TiDB Cloud, take the following steps: + +
    + 1. Open the **Import** page for your target cluster. 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. @@ -105,45 +109,102 @@ To import the Parquet files to TiDB Cloud, take the following steps: 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. -2. On the **Import** page: - - For a TiDB Dedicated cluster, click **Import Data** in the upper-right corner. - - For a TiDB Serverless cluster, click the **import data from S3** link above the upload area. +2. Select **Import data from S3**. -3. Provide the following information for the source Parquet files: + If this is your first time importing data into this cluster, select **Import From Amazon S3**. - - **Location**: select **Amazon S3**. - - **Data format**: select **Parquet**. - - **Bucket URI**: select the bucket URI where your Parquet files are located. Note that you must include `/` at the end of the URI, for example, `s3://sampledate/ingest/`. - - **Bucket Access** (This field is visible only for AWS S3): you can use either an AWS access key or a Role ARN to access your bucket. For more information, see [Configure Amazon S3 access](/tidb-cloud/config-s3-and-gcs-access.md#configure-amazon-s3-access). - - **AWS Access Keys**: enter the AWS access key ID and AWS secret access key. - - **AWS Role ARN**: enter the Role ARN value. +3. On the **Import Data from Amazon S3** page, provide the following information for the source Parquet files: -4. You can choose to **Import into Pre-created Tables**, or **Import Schema and Data from S3**. + - **Import File Count**: select **One file** or **Multiple files** as needed. + - **Included Schema Files**: this field is only visible when importing multiple files. If the source folder contains the target table schemas, select **Yes**. Otherwise, select **No**. + - **Data Format**: select **Parquet**. + - **File URI** or **Folder URI**: + - When importing one file, enter the source file URI and name in the following format `s3://[bucket_name]/[data_source_folder]/[file_name].parquet`. For example, `s3://sampledata/ingest/TableName.01.parquet`. + - When importing multiple files, enter the source file URI and name in the following format `s3://[bucket_name]/[data_source_folder]/`. For example, `s3://sampledata/ingest/`. + - **Bucket Access**: you can use either an AWS Role ARN or an AWS access key to access your bucket. For more information, see [Configure Amazon S3 access](/tidb-cloud/config-s3-and-gcs-access.md#configure-amazon-s3-access). + - **AWS Role ARN**: enter the AWS Role ARN value. + - **AWS Access Key**: enter the AWS access key ID and AWS secret access key. - - **Import into Pre-created Tables** allows you to create tables in TiDB in advance and select the tables that you want to import data into. In this case, you can choose up to 1000 tables to import. You can click **Chat2Qury** in the left navigation pane to create tables. For more information about how to use Chat2Qury, see [Explore Your Data with AI-Powered Chat2Query](/tidb-cloud/explore-data-with-chat2query.md). - - **Import Schema and Data from S3** allows you to import SQL scripts for creating a table and import corresponding table data stored in S3 into TiDB. +4. Click **Connect**. -5. If the source files do not meet the naming conventions, you can specify a custom mapping rule between a single target table and the CSV file. After that, the data source files will be re-scanned using the provided custom mapping rule. To modify the mapping, click **Advanced Settings** and then click **Mapping Settings**. Note that **Mapping Settings** is available only when you choose **Import into Pre-created Tables**. +5. In the **Destination** section, select the target database and table. - - **Target Database**: enter the name of the target database you select. + When importing multiple files, you can use **Advanced Settings** > **Mapping Settings** to define a custom mapping rule for each target table and its corresponding Parquet file. After that, the data source files will be re-scanned using the provided custom mapping rule. - - **Target Tables**: enter the name of the target table you select. Note that this field only accepts one specific table name, so wildcards are not supported. + When you enter the source file URI and name in **Source File URIs and Names**, make sure it is in the following format `s3://[bucket_name]/[data_source_folder]/[file_name].parquet`. For example, `s3://sampledata/ingest/TableName.01.parquet`. - - **Source file URIs and names**: enter the source file URI and name in the following format `s3://[bucket_name]/[data_source_folder]/[file_name].parquet`. For example, `s3://sampledate/ingest/TableName.01.parquet`. You can also use wildcards to match the source files. For example: + You can also use wildcards to match the source files. For example: - - `s3://[bucket_name]/[data_source_folder]/my-data?.parquet`: all Parquet files starting with `my-data` and one character (such as `my-data1.parquet` and `my-data2.parquet`) in that folder will be imported into the same target table. - - `s3://[bucket_name]/[data_source_folder]/my-data*.parquet`: all Parquet files in the folder starting with `my-data` will be imported into the same target table. + - `s3://[bucket_name]/[data_source_folder]/my-data?.parquet`: all Parquet files starting with `my-data` followed by one character (such as `my-data1.parquet` and `my-data2.parquet`) in that folder will be imported into the same target table. - Note that only `?` and `*` are supported. + - `s3://[bucket_name]/[data_source_folder]/my-data*.parquet`: all Parquet files in the folder starting with `my-data` will be imported into the same target table. - > **Note:** + Note that only `?` and `*` are supported. + + > **Note:** + > + > The URI must contain the data source folder. + +6. Click **Start Import**. + +7. When the import progress shows **Completed**, check the imported tables. + +
    + +
    + +1. Open the **Import** page for your target cluster. + + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** > - > The URI must contain the data source folder. + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + + 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +2. Click **Import Data** in the upper-right corner. + + If this is your first time importing data into this cluster, select **Import From GCS**. + +3. On the **Import Data from GCS** page, provide the following information for the source Parquet files: + + - **Import File Count**: select **One file** or **Multiple files** as needed. + - **Included Schema Files**: this field is only visible when importing multiple files. If the source folder contains the target table schemas, select **Yes**. Otherwise, select **No**. + - **Data Format**: select **Parquet**. + - **File URI** or **Folder URI**: + - When importing one file, enter the source file URI and name in the following format `gs://[bucket_name]/[data_source_folder]/[file_name].parquet`. For example, `gs://sampledata/ingest/TableName.01.parquet`. + - When importing multiple files, enter the source file URI and name in the following format `gs://[bucket_name]/[data_source_folder]/`. For example, `gs://sampledata/ingest/`. + - **Bucket Access**: you can use a GCS IAM Role to access your bucket. For more information, see [Configure GCS access](/tidb-cloud/config-s3-and-gcs-access.md#configure-gcs-access). + +4. Click **Connect**. + +5. In the **Destination** section, select the target database and table. + + When importing multiple files, you can use **Advanced Settings** > **Mapping Settings** to define a custom mapping rule for each target table and its corresponding Parquet file. After that, the data source files will be re-scanned using the provided custom mapping rule. + + When you enter the source file URI and name in **Source File URIs and Names**, make sure it is in the following format `gs://[bucket_name]/[data_source_folder]/[file_name].parquet`. For example, `gs://sampledata/ingest/TableName.01.parquet`. + + You can also use wildcards to match the source files. For example: + + - `gs://[bucket_name]/[data_source_folder]/my-data?.parquet`: all Parquet files starting with `my-data` followed by one character (such as `my-data1.parquet` and `my-data2.parquet`) in that folder will be imported into the same target table. + + - `gs://[bucket_name]/[data_source_folder]/my-data*.parquet`: all Parquet files in the folder starting with `my-data` will be imported into the same target table. + + Note that only `?` and `*` are supported. + + > **Note:** + > + > The URI must contain the data source folder. 6. Click **Start Import**. 7. When the import progress shows **Completed**, check the imported tables. +
    + +
    + When you run an import task, if any unsupported or invalid conversions are detected, TiDB Cloud terminates the import job automatically and reports an importing error. If you get an importing error, do the following: diff --git a/tidb-cloud/import-sample-data.md b/tidb-cloud/import-sample-data.md index e27ac45ced943..62d6d8ddd2732 100644 --- a/tidb-cloud/import-sample-data.md +++ b/tidb-cloud/import-sample-data.md @@ -7,6 +7,9 @@ summary: Learn how to import sample data into TiDB Cloud via UI. This document describes how to import the sample data into TiDB Cloud via the UI. The sample data used is the system data from Capital Bikeshare, released under the Capital Bikeshare Data License Agreement. Before importing the sample data, you need to have one TiDB cluster. + +
    + 1. Open the **Import** page for your target cluster. 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. @@ -17,41 +20,59 @@ This document describes how to import the sample data into TiDB Cloud via the UI 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. -2. Configure the source data information. +2. Select **Import data from S3**. + + If this is your first time importing data into this cluster, select **Import From Amazon S3**. - -
    +3. On the **Import Data from Amazon S3** page, configure the following source data information: - On the **Import** page: + - **Import File Count**: for the sample data, select **Multiple files**. + - **Included Schema Files**: for the sample data, select **Yes**. + - To import schema and data from the source, select **Yes**. This option imports SQL scripts for creating a table and import corresponding table data stored in S3 into TiDB. + - To import into pre-created tables, select **No**. This enables you to create tables in TiDB in advance and select the tables that you want to import data into. In this case, you can choose up to 1000 tables to import. You can click **SQL Editor** in the left navigation pane to create tables. For more information about how to use SQL Editor, see [Explore your data with AI-assisted SQL Editor](/tidb-cloud/explore-data-with-chat2query.md). + - **Data Format**: select **SQL**. TiDB Cloud supports importing compressed files in the following formats: `.gzip`, `.gz`, `.zstd`, `.zst` and `.snappy`. If you want to import compressed SQL files, name the files in the `${db_name}.${table_name}.${suffix}.sql.${compress}` format, in which `${suffix}` is optional and can be any integer such as '000001'. For example, if you want to import the `trips.000001.sql.gz` file to the `bikeshare.trips` table, you can rename the file as `bikeshare.trips.000001.sql.gz`. Note that you only need to compress the data files, not the database or table schema files. The Snappy compressed file must be in the [official Snappy format](https://github.com/google/snappy). Other variants of Snappy compression are not supported. + - **Folder URI** or **File URI**: enter the sample data URI `s3://tidbcloud-sample-data/data-ingestion/`. + - **Bucket Access**: for the sample data, you can only use a Role ARN to access its bucket. For your own data, you can use either an AWS access key or a Role ARN to access your bucket. + - **AWS Role ARN**: enter `arn:aws:iam::801626783489:role/import-sample-access`. + - **AWS Access Key**: skip this option for the sample data. - - For a TiDB Dedicated cluster, click **Import Data** in the upper-right corner. - - For a TiDB Serverless cluster, click the **import data from S3** link above the upload area. + If the region of the bucket is different from your cluster, confirm the compliance of cross region. - Fill in the following parameters: +4. Click **Connect** > **Start Import**. - - **Data format**: select **SQL File**. TiDB Cloud supports importing compressed files in the following formats: `.gzip`, `.gz`, `.zstd`, `.zst` and `.snappy`. If you want to import compressed SQL files, name the files in the `${db_name}.${table_name}.${suffix}.sql.${compress}` format, in which `${suffix}` is optional and can be any integer such as '000001'. For example, if you want to import the `trips.000001.sql.gz` file to the `bikeshare.trips` table, you can rename the file as `bikeshare.trips.000001.sql.gz`. Note that you only need to compress the data files, not the database or table schema files. - - **Bucket URI**: enter the sample data URI `s3://tidbcloud-sample-data/data-ingestion/` - - **Bucket Access**: for the sample data, you can only use a Role ARN to access its bucket. For your own data, you can use either an AWS access key or a Role ARN to access your bucket. - - **AWS Access Keys**: skip this option for the sample data. - - **AWS Role ARN**: enter `arn:aws:iam::801626783489:role/import-sample-access` +
    +
    - If the region of the bucket is different from your cluster, confirm the compliance of cross region. Click **Next**. +1. Open the **Import** page for your target cluster. + + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + + 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. -
    +2. Click **Import Data** in the upper-right corner. -
    + If this is your first time importing data into this cluster, select **Import From GCS**. - If your TiDB cluster is hosted by Google Cloud, click **Import Data** in the upper-right corner, and then fill in the following parameters: +3. On the **Import Data from GCS** page, configure the following source data information: - - **Data format**: select **SQL File**. TiDB Cloud supports importing compressed files in the following formats: `.gzip`, `.gz`, `.zstd`, `.zst` and `.snappy`. If you want to import compressed SQL files, name the files in the `${db_name}.${table_name}.${suffix}.sql.${compress}` format, in which `${suffix}` is optional and can be any integer such as '000001'. For example, if you want to import the `trips.000001.sql.gz` file to the `bikeshare.trips` table, you can rename the file as `bikeshare.trips.000001.sql.gz`. Note that you only need to compress the data files, not the database or table schema files. - - **Bucket URI**: enter the sample data URI `gs://tidbcloud-samples-us-west1`. + - **Import File Count**: for the sample data, select **Multiple files**. + - **Included Schema Files**: for the sample data, select **Yes**. + - To import schema and data from the source, select **Yes**. This option imports SQL scripts for creating a table and import corresponding table data stored in S3 into TiDB. + - To import into pre-created tables, select **No**. This enables you to create tables in TiDB in advance and select the tables that you want to import data into. In this case, you can choose up to 1000 tables to import. You can click **SQL Editor** in the left navigation pane to create tables. For more information about how to use SQL Editor, see [Explore your data with AI-assisted SQL Editor](/tidb-cloud/explore-data-with-chat2query.md). + - **Data Format**: select **SQL**. TiDB Cloud supports importing compressed files in the following formats: `.gzip`, `.gz`, `.zstd`, `.zst` and `.snappy`. If you want to import compressed SQL files, name the files in the `${db_name}.${table_name}.${suffix}.sql.${compress}` format, in which `${suffix}` is optional and can be any integer such as '000001'. For example, if you want to import the `trips.000001.sql.gz` file to the `bikeshare.trips` table, you can rename the file as `bikeshare.trips.000001.sql.gz`. Note that you only need to compress the data files, not the database or table schema files. Note that you only need to compress the data files, not the database or table schema files. The Snappy compressed file must be in the [official Snappy format](https://github.com/google/snappy). Other variants of Snappy compression are not supported. + - **Folder URI** or **File URI**: enter the sample data URI `gs://tidbcloud-samples-us-west1/`. + - **Bucket Access**: you can use a GCS IAM Role to access your bucket. For more information, see [Configure GCS access](/tidb-cloud/config-s3-and-gcs-access.md#configure-gcs-access). - If the region of the bucket is different from your cluster, confirm the compliance of cross region. Click **Next**. + If the region of the bucket is different from your cluster, confirm the compliance of cross region. -
    -
    +4. Click **Connect** > **Start Import**. -3. Click **Start Import**. +
    +
    When the data import progress shows **Completed**, you have successfully imported the sample data and the database schema to your database in TiDB Cloud. diff --git a/tidb-cloud/import-snapshot-files.md b/tidb-cloud/import-snapshot-files.md index 9d0e01d63d7d8..e31daac06fc85 100644 --- a/tidb-cloud/import-snapshot-files.md +++ b/tidb-cloud/import-snapshot-files.md @@ -7,4 +7,4 @@ summary: Learn how to import Amazon Aurora or RDS for MySQL snapshot files into You can import Amazon Aurora or RDS for MySQL snapshot files into TiDB Cloud. Note that all source data files with the `.parquet` suffix in the `{db_name}.{table_name}/` folder must conform to the [naming convention](/tidb-cloud/naming-conventions-for-data-import.md). -The process of importing snapshot files is similiar to that of importing Parquet files. For more information, see [Import Apache Parquet Files from Amazon S3 or GCS into TiDB Cloud](/tidb-cloud/import-parquet-files.md). +The process of importing snapshot files is similar to that of importing Parquet files. For more information, see [Import Apache Parquet Files from Amazon S3 or GCS into TiDB Cloud](/tidb-cloud/import-parquet-files.md). diff --git a/tidb-cloud/import-with-mysql-cli.md b/tidb-cloud/import-with-mysql-cli.md new file mode 100644 index 0000000000000..fe9cc7c22f5e2 --- /dev/null +++ b/tidb-cloud/import-with-mysql-cli.md @@ -0,0 +1,128 @@ +--- +title: Import Data into TiDB Cloud via MySQL CLI +summary: Learn how to import Data into TiDB Cloud via MySQL CLI. +--- + +# Import Data into TiDB Cloud via MySQL CLI + +This document describes how to import data into TiDB Cloud via the [MySQL Command-Line Client](https://dev.mysql.com/doc/refman/8.0/en/mysql.html). You can import data from an SQL file or a CSV file. The following sections provide step-by-step instructions for importing data from each type of file. + +## Prerequisites + +Before you can import data via MySQL CLI to TiDB Cloud, you need the following prerequisites: + +- You have access to your TiDB Cloud cluster. If you do not have a TiDB cluster, create one following the instructions in [Build a TiDB Cloud Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). +- Install MySQL CLI on your local computer. + +## Step 1. Connect to your TiDB Cloud cluster + +Connect to your TiDB cluster depending on the TiDB deployment option you have selected. + + +
    + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Ensure the configurations in the connection dialog match your operating environment. + + - **Connection Type** is set to `Public`. + - **Connect With** is set to `MySQL CLI`. + - **Operating System** matches your environment. + +4. Click **Generate Password** to create a random password. + + > **Tip:** + > + > If you have created a password before, either use the original password or click **Reset Password** to generate a new one. + +
    +
    + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Click **Allow Access from Anywhere**. + + For more details about how to obtain the connection string, see [Connect to TiDB Cloud Dedicated via Public Connection](/tidb-cloud/connect-via-standard-connection.md). + +
    +
    + +## Step 2. Define the table and insert sample data + +Before importing data, you need to prepare the table structure and insert real sample data into it. The following is an example SQL file (`product_data.sql`) that you can use to create a table and insert sample data: + +```sql +-- Create a table in your TiDB database +CREATE TABLE products ( + product_id INT PRIMARY KEY, + product_name VARCHAR(255), + price DECIMAL(10, 2) +); + +-- Insert sample data into the table +INSERT INTO products (product_id, product_name, price) VALUES + (1, 'Laptop', 999.99), + (2, 'Smartphone', 499.99), + (3, 'Tablet', 299.99); +``` + +## Step 3. Import data from a SQL or CSV file + +You can import data from an SQL file or a CSV file. The following sections provide step-by-step instructions for importing data from each type. + + +
    + +Do the following to import data from an SQL file: + +1. Provide a real SQL file (for example, `product_data.sql`) that contains the data you want to import. This SQL file must contain `INSERT` statements with real data. + +2. Use the following command to import data from the SQL file: + + ```bash + mysql --comments --connect-timeout 150 -u '' -h -P 4000 -D test --ssl-mode=VERIFY_IDENTITY --ssl-ca= -p < product_data.sql + ``` + +> **Note:** +> +> The default database name used here is `test`, and you can either manually create your own database or use the `CREATE DATABASE` command in an SQL file. + +
    +
    + +Do the following to import data from a CSV file: + +1. Create a database and schema in TiDB to match your data import needs. + +2. Provide a sample CSV file (for example, `product_data.csv`) that contains the data you want to import. The following is an example of a CSV file: + + **product_data.csv:** + + ```csv + product_id,product_name,price + 4,Laptop,999.99 + 5,Smartphone,499.99 + 6,Tablet,299.99 + ``` + +3. Use the following command to import data from the CSV file: + + ```bash + mysql --comments --connect-timeout 150 -u '' -h -P 4000 -D test --ssl-mode=VERIFY_IDENTITY --ssl-ca= -p -e "LOAD DATA LOCAL INFILE '' INTO TABLE products + FIELDS TERMINATED BY ',' + LINES TERMINATED BY '\n' + IGNORE 1 LINES (product_id, product_name, price);" + ``` + +4. Make sure to replace the paths, table name (`products` in this example), ``, ``, ``, ``, ``, and other placeholders with your actual information, and replace the sample CSV data with your real dataset as needed. + +> **Note:** +> +> For more syntax details about `LOAD DATA LOCAL INFILE`, see [`LOAD DATA`](/sql-statements/sql-statement-load-data.md). + +
    +
    diff --git a/tidb-cloud/index-insight.md b/tidb-cloud/index-insight.md index 5557e1e51f107..4aadef56e36da 100644 --- a/tidb-cloud/index-insight.md +++ b/tidb-cloud/index-insight.md @@ -9,7 +9,7 @@ The Index Insight (beta) feature in TiDB Cloud provides powerful capabilities to > **Note:** > -> Index Insight is currently in beta and only available for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. +> Index Insight is currently in beta and only available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters. ## Introduction @@ -26,11 +26,11 @@ This section introduces how to enable the Index Insight feature and obtain recom ### Before you begin -Before enabling the Index Insight feature, make sure that you have created a TiDB Dedicated cluster. If you do not have one, follow the steps in [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md) to create one. +Before enabling the Index Insight feature, make sure that you have created a TiDB Cloud Dedicated cluster. If you do not have one, follow the steps in [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md) to create one. ### Step 1: Enable Index Insight -1. In the [TiDB Cloud console](https://tidbcloud.com), navigate to the cluster overview page of your TiDB Dedicated cluster, and then click **Diagnosis** in the left navigation pane. +1. In the [TiDB Cloud console](https://tidbcloud.com), navigate to the cluster overview page of your TiDB Cloud Dedicated cluster, and then click **Diagnosis** in the left navigation pane. 2. Click the **Index Insight BETA** tab. The **Index Insight overview** page is displayed. @@ -46,7 +46,7 @@ Before enabling the Index Insight feature, make sure that you have created a TiD > **Note:** > - > To connect to your TiDB Dedicated cluster, see [Connect to a TiDB Dedicated cluster](/tidb-cloud/connect-to-tidb-cluster.md). + > To connect to your TiDB Cloud Dedicated cluster, see [Connect to a TiDB Cloud Dedicated cluster](/tidb-cloud/connect-to-tidb-cluster.md). 4. Enter the username and password of the SQL user created in the preceding step. Then, click **Activate** to initiate the activation process. diff --git a/tidb-cloud/integrate-tidbcloud-with-airbyte.md b/tidb-cloud/integrate-tidbcloud-with-airbyte.md index c6ace16dfe33b..eb21250caa2d8 100644 --- a/tidb-cloud/integrate-tidbcloud-with-airbyte.md +++ b/tidb-cloud/integrate-tidbcloud-with-airbyte.md @@ -26,7 +26,7 @@ You can deploy Airbyte locally with only a few steps. docker-compose up ``` -Once you see an Airbyte banner, you can go to with the username (`airbyte`) and password (`password`) to visit the UI. +Once you see an Airbyte banner, you can go to [http://localhost:8000](http://localhost:8000) with the username (`airbyte`) and password (`password`) to visit the UI. ``` airbyte-server | ___ _ __ __ @@ -62,7 +62,7 @@ Conveniently, the steps are the same for setting TiDB as the source and the dest > > - TiDB Cloud supports TLS connection. You can choose your TLS protocols in **TLSv1.2** and **TLSv1.3**, for example, `enabledTLSProtocols=TLSv1.2`. > - If you want to disable TLS connection to TiDB Cloud via JDBC, you need to set useSSL to `false` in JDBC URL Params specifically and close SSL connection, for example, `useSSL=false`. - > - TiDB Serverless only supports TLS connections. + > - TiDB Cloud Serverless only supports TLS connections. 4. Click **Set up source** or **destination** to complete creating the connector. The following screenshot shows the configuration of TiDB as the source. @@ -87,7 +87,7 @@ The following steps use TiDB as both a source and a destination. Other connector > **Tip:** > - > The TiDB connector supports both Incremental and Full Refresh syncs. + > The TiDB connector supports both [Incremental and Full Refresh syncs](https://airbyte.com/blog/understanding-data-replication-modes). > > - In Incremental mode, Airbyte only reads records added to the source since the last sync job. The first sync using Incremental mode is equivalent to Full Refresh mode. > - In Full Refresh mode, Airbyte reads all records in the source and replicates to the destination in every sync task. You can set the sync mode for every table named **Namespace** in Airbyte individually. @@ -102,7 +102,7 @@ The following steps use TiDB as both a source and a destination. Other connector ## Limitations -- The TiDB connector does not support the Change Data Capture (CDC) feature. +- The TiDB connector cannot use the Change Data Capture (CDC) feature provided by TiCDC. The incremental sync is performed based on a cursor mechanism. - TiDB destination converts the `timestamp` type to the `varchar` type in default normalization mode. It happens because Airbyte converts the timestamp type to string during transmission, and TiDB does not support `cast ('2020-07-28 14:50:15+1:00' as timestamp)`. - For some large ELT missions, you need to increase the parameters of [transaction restrictions](/develop/dev-guide-transaction-restraints.md#large-transaction-restrictions) in TiDB. diff --git a/tidb-cloud/integrate-tidbcloud-with-cloudflare.md b/tidb-cloud/integrate-tidbcloud-with-cloudflare.md index 195a561ff6c42..c9d78aaea21c9 100644 --- a/tidb-cloud/integrate-tidbcloud-with-cloudflare.md +++ b/tidb-cloud/integrate-tidbcloud-with-cloudflare.md @@ -1,192 +1,102 @@ --- title: Integrate TiDB Cloud with Cloudflare -summary: Learn how deploy Cloudflare Workers with TiDB Cloud. +summary: Learn how to deploy Cloudflare Workers with TiDB Cloud. --- # Integrate TiDB Cloud with Cloudflare Workers [Cloudflare Workers](https://workers.cloudflare.com/) is a platform that allows you to run code in response to specific events, such as HTTP requests or changes to a database. Cloudflare Workers is easy to use and can be used to build a variety of applications, including custom APIs, serverless functions, and microservices. It is particularly useful for applications that require low-latency performance or need to scale quickly. -However, you may find it hard to connect to TiDB Cloud from Cloudflare Workers because Cloudflare Workers runs on the V8 engine which cannot make direct TCP connections. +You may find it hard to connect to TiDB Cloud from Cloudflare Workers because Cloudflare Workers runs on the V8 engine which cannot make direct TCP connections. You can use [TiDB Cloud serverless driver](/tidb-cloud/serverless-driver.md) to help you connect to Cloudflare Workers over HTTP connection. -Fortunately, Prisma has your back with the [Data Proxy](https://www.prisma.io/docs/data-platform/data-proxy). It can help you use Cloudflare Workers to process and manipulate the data being transmitted over a TCP connection. - -This document shows how to deploy Cloudflare Workers with TiDB Cloud and Prisma Data Proxy step by step. +This document shows how to connect to Cloudflare Workers with TiDB Cloud serverless driver step by step. > **Note:** > -> If you want to connect a locally deployed TiDB to Cloudflare Workers, you can try [worker-tidb](https://github.com/shiyuhang0/worker-tidb), which uses Cloudflare tunnels as a proxy. However, worker-tidb is not recommended for production use. +> TiDB Cloud serverless driver can only be used in TiDB Cloud Serverless. ## Before you begin Before you try the steps in this article, you need to prepare the following things: -- A TiDB Cloud account and a TiDB Serverless cluster on TiDB Cloud. For more details, see [TiDB Cloud Quick Start](/tidb-cloud/tidb-cloud-quickstart.md#step-1-create-a-tidb-cluster). +- A TiDB Cloud account and a TiDB Cloud Serverless cluster on TiDB Cloud. For more details, see [TiDB Cloud Quick Start](/tidb-cloud/tidb-cloud-quickstart.md#step-1-create-a-tidb-cluster). - A [Cloudflare Workers account](https://dash.cloudflare.com/login). -- A [Prisma Data Platform account](https://cloud.prisma.io/). -- A [GitHub account](https://github.com/login). -- Install Node.js and npm. -- Install dependencies using `npm install -D prisma typescript wrangler` +- [npm](https://docs.npmjs.com/about-npm) is installed. ## Step 1: Set up Wrangler [Wrangler](https://developers.cloudflare.com/workers/wrangler/) is the official Cloudflare Worker CLI. You can use it to generate, build, preview, and publish your Workers. -1. To authenticate Wrangler, run wrangler login: +1. Install Wrangler: + + ``` + npm install wrangler + ``` + +2. To authenticate Wrangler, run wrangler login: ``` wrangler login ``` -2. Use Wrangler to create a worker project: +3. Use Wrangler to create a worker project: ``` - wrangler init prisma-tidb-cloudflare + wrangler init tidb-cloud-cloudflare ``` -3. In your terminal, you will be asked a series of questions related to your project. Choose the default values for all questions. +4. In your terminal, you will be asked a series of questions related to your project. Choose the default values for all questions. -## Step 2: Set up Prisma +## Step 2: Install the serverless driver 1. Enter your project directory: ``` - cd prisma-tidb-cloudflare - ``` - -2. Use the `prisma init` command to set up Prisma: - - ``` - npx prisma init + cd tidb-cloud-cloudflare ``` - This creates a Prisma schema in `prisma/schema.prisma`. - -3. Inside `prisma/schema.prisma`, add the schema according to your tables in TiDB. Assume that you have `table1` and `table2` in TiDB, you can add the following schema: +2. Install the serverless driver with npm: ``` - generator client { - provider = "prisma-client-js" - } - - datasource db { - provider = "mysql" - url = env("DATABASE_URL") - } - - model table1 { - id Int @id @default(autoincrement()) - name String - } - - model table2 { - id Int @id @default(autoincrement()) - name String - } + npm install @tidbcloud/serverless ``` - This data model will be used to store incoming requests from your Worker. - -## Step 3: Push your project to GitHub + This adds the serverless driver dependency in `package.json`. -1. [Create a repository](https://github.com/new) named `prisma-tidb-cloudflare` on GitHub. +## Step 3: Develop the Cloudflare Worker function -2. After you create the repository, you can push your project to GitHub: - - ``` - git remote add origin https://github.com//prisma-tidb-cloudflare - git add . - git commit -m "initial commit" - git push -u origin main - ``` +You need to modify the `src/index.ts` according to your needs. -## Step 4: Import your Project into the Prisma Data Platform - -With Cloudflare Workers, you cannot directly access your database because there is no TCP support. Instead, you can use Prisma Data Proxy as described above. - -1. To get started, sign in to the [Prisma Data Platform](https://cloud.prisma.io/) and click **New Project**. -2. Fill in the **Connection string** with this pattern `mysql://USER:PASSWORD@HOST:PORT/DATABASE?sslaccept=strict`. You can find the connection information in your [TiDB Cloud console](https://tidbcloud.com/console/clusters). -3. Leave the **Static IPs** as disabled because TiDB Serverless is accessible from any IP address. -4. Select a Data Proxy region that is geographically close to your TiDB Cloud cluster location. Then click **Create project**. - - ![Configure project settings](/media/tidb-cloud/cloudflare/cloudflare-project.png) - -5. Fill in the repository, and click **Link Prisma schema** in the **Get Started** page. -6. Click **Create a new connection string** and you will get a new connection string that starts with `prisma://.` Copy this connection string and save it for later. - - ![Create new connection string](/media/tidb-cloud/cloudflare/cloudflare-start.png) - -7. Click **Skip and continue to Data Platform** to go to the Data Platform. - -## Step 5: Set the Data Proxy Connection string in your environment - -1. Add the Data Proxy connection string to your local environment `.env` file: - - ``` - DATABASE_URL=prisma://aws-us-east-1.prisma-data.com/?api_key=•••••••••••••••••" - ``` +For example, if you want to show all the databases, you can use the following code: -2. Add the Data Proxy connection to Cloudflare Workers with secret: +```ts +import { connect } from '@tidbcloud/serverless' - ``` - wrangler secret put DATABASE_URL - ``` -3. According to the prompt, enter the Data Proxy connection string. +export interface Env { + DATABASE_URL: string; +} -> **Note:** -> -> You can also edit the `DATABASE_URL` secret via the Cloudflare Workers dashboard. +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { + const conn = connect({url:env.DATABASE_URL}) + const resp = await conn.execute("show databases") + return new Response(JSON.stringify(resp)); + }, +}; +``` -## Step 6: Generate a Prisma Client +## Step 4: Set the DATABASE_URL in your environment -Generate a Prisma Client that connects through the [Data Proxy](https://www.prisma.io/docs/data-platform/data-proxy): +The `DATABASE_URL` follows the `mysql://username:password@host/database` format. You can set the environment variable with wrangler cli: ``` -npx prisma generate --data-proxy +wrangler secret put ``` -## Step 7: Develop the Cloudflare Worker function - -You need to change the `src/index.ts` according to your needs. - -For example, if you want to query different tables with an URL variable, you can use the following code: - -```js -import { PrismaClient } from '@prisma/client/edge' -const prisma = new PrismaClient() - -addEventListener('fetch', (event) => { - event.respondWith(handleEvent(event)) -}) - -async function handleEvent(event: FetchEvent): Promise { - // Get URL parameters - const { request } = event - const url = new URL(request.url); - const table = url.searchParams.get('table'); - let limit = url.searchParams.get('limit'); - const limitNumber = limit? parseInt(limit): 100; - - // Get model - let model - for (const [key, value] of Object.entries(prisma)) { - if (typeof value == 'object' && key == table) { - model = value - break - } - } - if(!model){ - return new Response("Table not defined") - } - - // Get data - const result = await model.findMany({ take: limitNumber }) - return new Response(JSON.stringify({ result })) -} -``` +You can also edit the `DATABASE_URL` secret via the Cloudflare Workers dashboard. -## Step 8: Publish to Cloudflare Workers +## Step 5: Publish to Cloudflare Workers You're now ready to deploy to Cloudflare Workers. @@ -196,39 +106,12 @@ In your project directory, run the following command: npx wrangler publish ``` -## Step 9: Try your Cloudflare Workers +## Step 6: Try your Cloudflare Workers 1. Go to [Cloudflare dashboard](https://dash.cloudflare.com) to find your worker. You can find the URL of your worker on the overview page. -2. Visit the URL with your table name: `https://{your-worker-url}/?table={table_name}`. You will get the result from the corresponding TiDB table. - -## Update the project - -### Change the serverless function - -If you want to change the serverless function, update `src/index.ts` and publish it to Cloudflare Workers again. - -### Create a new table +2. Visit the URL and you will get the result. -If you create a new table and want to query it, take the following steps: +## Examples -1. Add a new model in `prisma/schema.prisma`. -2. Push the changes to your repository. - - ``` - git add prisma - git commit -m "add new model" - git push - ``` - -3. Generate the Prisma Client again. - - ``` - npx prisma generate --data-proxy - ``` - -4. Publish the Cloudflare Worker again. - - ``` - npx wrangler publish - ``` +See the [Cloudflare Workers example](https://github.com/tidbcloud/car-sales-insight/tree/main/examples/cloudflare-workers). \ No newline at end of file diff --git a/tidb-cloud/integrate-tidbcloud-with-dbt.md b/tidb-cloud/integrate-tidbcloud-with-dbt.md index ab90fa8111b71..5debd3c0a8c30 100644 --- a/tidb-cloud/integrate-tidbcloud-with-dbt.md +++ b/tidb-cloud/integrate-tidbcloud-with-dbt.md @@ -315,7 +315,7 @@ To generate visual documents, take the following steps: dbt docs serve ``` -3. To access the document from your browser, go to . +3. To access the document from your browser, go to [http://localhost:8080](http://localhost:8080). ## Description of profile fields diff --git a/tidb-cloud/integrate-tidbcloud-with-n8n.md b/tidb-cloud/integrate-tidbcloud-with-n8n.md index e815286686217..bf5fb005b2014 100644 --- a/tidb-cloud/integrate-tidbcloud-with-n8n.md +++ b/tidb-cloud/integrate-tidbcloud-with-n8n.md @@ -7,7 +7,7 @@ summary: Learn the use of TiDB Cloud node in n8n. [n8n](https://n8n.io/) is an extendable workflow automation tool. With a [fair-code](https://faircode.io/) distribution model, n8n will always have visible source code, be available to self-host, and allow you to add your custom functions, logic, and apps. -This document introduces how to build an auto-workflow: create a TiDB Serverless cluster, gather Hacker News RSS, store it to TiDB and send a briefing email. +This document introduces how to build an auto-workflow: create a TiDB Cloud Serverless cluster, gather Hacker News RSS, store it to TiDB and send a briefing email. ## Prerequisites: Get TiDB Cloud API key @@ -76,9 +76,9 @@ The final workflow should look like the following image. ![img](/media/tidb-cloud/integration-n8n-workflow-rss.jpg) -### (Optional) Create a TiDB Serverless cluster +### (Optional) Create a TiDB Cloud Serverless cluster -If you don't have a TiDB Serverless cluster, you can use this node to create one. Otherwise, feel free to skip this operation. +If you don't have a TiDB Cloud Serverless cluster, you can use this node to create one. Otherwise, feel free to skip this operation. 1. Navigate to **Workflows** panel, and click **Add workflow**. 2. In new workflow workspace, click **+** in the top right corner and choose **All** field. @@ -93,7 +93,7 @@ If you don't have a TiDB Serverless cluster, you can use this node to create one > **Note:** > -> It takes several seconds to create a new TiDB Serverless cluster. +> It takes several seconds to create a new TiDB Cloud Serverless cluster. ### Create a workflow @@ -203,7 +203,7 @@ This trigger will execute your workflow every morning at 8 AM. After building up the workflow, you can click **Execute Workflow** to test run it. -If the workflow runs as expected, you'll get Hacker News briefing emails. These news contents will be logged to your TiDB Serverless cluster, so you don't have to worry about losing them. +If the workflow runs as expected, you'll get Hacker News briefing emails. These news contents will be logged to your TiDB Cloud Serverless cluster, so you don't have to worry about losing them. Now you can activate this workflow in the **Workflows** panel. This workflow will help you get the front-page articles on Hacker News every day. @@ -213,7 +213,7 @@ Now you can activate this workflow in the **Workflows** panel. This workflow wil TiDB Cloud node acts as a [regular node](https://docs.n8n.io/workflows/nodes/#regular-nodes) and only supports the following five operations: -- **Create Serverless Cluster**: creates a TiDB Serverless cluster. +- **Create Serverless Cluster**: creates a TiDB Cloud Serverless cluster. - **Execute SQL**: executes an SQL statement in TiDB. - **Delete**: deletes rows in TiDB. - **Insert**: inserts rows in TiDB. diff --git a/tidb-cloud/integrate-tidbcloud-with-netlify.md b/tidb-cloud/integrate-tidbcloud-with-netlify.md index 60e252ccef185..ab13a4267cdd8 100644 --- a/tidb-cloud/integrate-tidbcloud-with-netlify.md +++ b/tidb-cloud/integrate-tidbcloud-with-netlify.md @@ -7,7 +7,7 @@ summary: Learn how to connect your TiDB Cloud clusters to Netlify projects. [Netlify](https://netlify.com/) is an all-in-one platform for automating modern web projects. It replaces your hosting infrastructure, continuous integration, and deployment pipeline with a single workflow and integrates dynamic functionality like serverless functions, user authentication, and form handling as your projects grow. -This document describes how to deploy a fullstack app on Netlify with TiDB Cloud as the database backend. +This document describes how to deploy a fullstack app on Netlify with TiDB Cloud as the database backend. You can also learn how to use Netlify edge function with our TiDB Cloud serverless driver. ## Prerequisites @@ -24,16 +24,16 @@ You are expected to have a Netlify account and CLI. If you do not have any, refe You are expected to have an account and a cluster in TiDB Cloud. If you do not have any, refer to the following to create one: -- [Create a TiDB Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) -- [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md) +- [Create a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) +- [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md) One TiDB Cloud cluster can connect to multiple Netlify sites. ### All IP addresses allowed for traffic filter in TiDB Cloud -For TiDB Dedicated clusters, make sure that the traffic filter of the cluster allows all IP addresses (set to `0.0.0.0/0`) for connection. This is because Netlify deployments use dynamic IP addresses. +For TiDB Cloud Dedicated clusters, make sure that the traffic filter of the cluster allows all IP addresses (set to `0.0.0.0/0`) for connection. This is because Netlify deployments use dynamic IP addresses. -TiDB Serverless clusters allow all IP addresses for connection by default, so you do not need to configure any traffic filter. +TiDB Cloud Serverless clusters allow all IP addresses for connection by default, so you do not need to configure any traffic filter. ## Step 1. Get the example project and the connection string @@ -52,9 +52,9 @@ To help you get started quickly, TiDB Cloud provides a fullstack example app in ### Get the TiDB Cloud connection string -For a TiDB Serverless cluster, you can get the connection string either from [TiDB Cloud CLI](/tidb-cloud/cli-reference.md) or from [TiDB Cloud console](https://tidbcloud.com/). +For a TiDB Cloud Serverless cluster, you can get the connection string either from [TiDB Cloud CLI](/tidb-cloud/cli-reference.md) or from [TiDB Cloud console](https://tidbcloud.com/). -For a TiDB Dedicated cluster, you can get the connection string only from the TiDB Cloud console. +For a TiDB Cloud Dedicated cluster, you can get the connection string only from the TiDB Cloud console.
    @@ -82,7 +82,7 @@ For a TiDB Dedicated cluster, you can get the connection string only from the Ti The output is as follows, where you can find the connection string for Prisma in the `url` value. - ``` + ```shell datasource db { provider = "mysql" url = "mysql://:@:/?sslaccept=strict" @@ -108,7 +108,7 @@ For a TiDB Dedicated cluster, you can get the connection string only from the Ti 2. Fill the connection parameters in the following connection string: - ``` + ```shell mysql://:@:/?sslaccept=strict ``` @@ -222,4 +222,40 @@ For a TiDB Dedicated cluster, you can get the connection string only from the Ti netlify deploy --prod --trigger ``` - Go to your Netlify console to check the deployment state. After the deployment is done, the site for the app will have a public IP address provided by Netlify so that everyone can access it. \ No newline at end of file + Go to your Netlify console to check the deployment state. After the deployment is done, the site for the app will have a public IP address provided by Netlify so that everyone can access it. + +## Use the edge function + +The example app mentioned in the section above runs on the Netlify serverless function. This section shows you how to use the edge function with [TiDB Cloud serverless driver](/tidb-cloud/serverless-driver.md). The edge function is a feature provided by Netlify, which allows you to run serverless functions on the edge of the Netlify CDN. + +To use the edge function, take the following steps: + +1. Create a directory named `netlify/edge-functions` in the root directory of your project. + +2. Create a file named `hello.ts` in the directory and add the following code: + + ```typescript + import { connect } from 'https://esm.sh/@tidbcloud/serverless' + + export default async () => { + const conn = connect({url: Netlify.env.get('DATABASE_URL')}) + const result = await conn.execute('show databases') + return new Response(JSON.stringify(result)); + } + + export const config = { path: "/api/hello" }; + ``` + +3. Set the `DATABASE_URL` environment variables. You can get the connection information from the [TiDB Cloud console](https://tidbcloud.com/). + + ```shell + netlify env:set DATABASE_URL 'mysql://:@/' + ``` + +4. Deploy the edge function to Netlify. + + ```shell + netlify deploy --prod --trigger + ``` + +Then you can go to your Netlify console to check the state of the deployment. After the deployment is done, you can access the edge function through the `https:///api/hello` URL. diff --git a/tidb-cloud/integrate-tidbcloud-with-vercel.md b/tidb-cloud/integrate-tidbcloud-with-vercel.md index c5132ad1671a1..f576c97d633e5 100644 --- a/tidb-cloud/integrate-tidbcloud-with-vercel.md +++ b/tidb-cloud/integrate-tidbcloud-with-vercel.md @@ -18,7 +18,7 @@ This guide describes how to connect your TiDB Cloud clusters to Vercel projects For both of the preceding methods, TiDB Cloud provides the following options for programmatically connecting to your database: -- Direct connection: connect your TiDB Cloud cluster to your Vercel project directly using MySQL's standard connection system. +- Cluster: connect your TiDB Cloud cluster to your Vercel project with direct connections or [serverless driver](/tidb-cloud/serverless-driver.md). - [Data App](/tidb-cloud/data-service-manage-data-app.md): access data of your TiDB Cloud cluster through a collection of HTTP endpoints. ## Prerequisites @@ -38,17 +38,17 @@ One Vercel project can only connect to one TiDB Cloud cluster. To change the int You are expected to have an account and a cluster in TiDB Cloud. If you do not have any, refer to the following to create one: -- [Create a TiDB Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) +- [Create a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) > **Note:** > - > The TiDB Cloud Vercel integration supports creating TiDB Serverless clusters. You can also create one later during the integration process. + > The TiDB Cloud Vercel integration supports creating TiDB Cloud Serverless clusters. You can also create one later during the integration process. -- [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md) +- [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md) > **Note:** > - > For TiDB Dedicated clusters, make sure that the traffic filter of the cluster allows all IP addresses (set to `0.0.0.0/0`) for connection, because Vercel deployments use [dynamic IP addresses](https://vercel.com/guides/how-to-allowlist-deployment-ip-address). If you use the TiDB Cloud Vercel integration, TiDB Cloud automatically adds a `0.0.0.0/0` traffic filter to your cluster in the integration workflow if there is none. + > For TiDB Cloud Dedicated clusters, make sure that the traffic filter of the cluster allows all IP addresses (set to `0.0.0.0/0`) for connection, because Vercel deployments use [dynamic IP addresses](https://vercel.com/guides/how-to-allowlist-deployment-ip-address). To [integrate with Vercel via the TiDB Cloud Vercel Integration](#connect-via-the-tidb-cloud-vercel-integration), you are expected to be in the `Organization Owner` role of your organization or the `Project Owner` role of the target project in TiDB Cloud. For more information, see [User roles](/tidb-cloud/manage-user-access.md#user-roles). @@ -69,10 +69,16 @@ One Vercel project can only connect to one TiDB Cloud Data App. To change the Da To connect via the TiDB Cloud Vercel integration, go to the [TiDB Cloud integration](https://vercel.com/integrations/tidb-cloud) page from the [Vercel's Integrations Marketplace](https://vercel.com/integrations). Using this method, you can choose which cluster to connect to, and TiDB Cloud will automatically generate all the necessary environment variables for your Vercel projects. +> **Note:** +> +> This method is only available for TiDB Cloud Serverless clusters. If you want to connect to a TiDB Cloud Dedicated cluster, use the [manual method](#connect-via-manually-setting-environment-variables). + +### Integration workflow + The detailed steps are as follows: -
    +
    1. Click **Add Integration** in the upper-right area of the [TiDB Cloud Vercel integration](https://vercel.com/integrations/tidb-cloud) page. The **Add TiDB Cloud** dialog is displayed. 2. Select the scope of your integration in the drop-down list and click **Continue**. @@ -83,9 +89,11 @@ The detailed steps are as follows: 1. Select your target Vercel projects and click **Next**. 2. Select your target TiDB Cloud organization and project. 3. Select **Cluster** as your connection type. - 4. Select your target TiDB Cloud cluster. If the **Cluster** drop-down list is empty or you want to select a new TiDB Serverless cluster, click **+ Create Cluster** in the list to create one. - 5. Select the framework that your Vercel projects are using. If the target framework is not listed, select **General**. Different frameworks determine different environment variables. - 6. Click **Add Integration and Return to Vercel**. + 4. Select your target TiDB Cloud cluster. If the **Cluster** drop-down list is empty or you want to select a new TiDB Cloud Serverless cluster, click **+ Create Cluster** in the list to create one. + 5. Select the database that you want to connect to. If the **Database** drop-down list is empty or you want to select a new Database, click **+ Create Database** in the list to create one. + 6. Select the framework that your Vercel projects are using. If the target framework is not listed, select **General**. Different frameworks determine different environment variables. + 7. Choose whether to enable **Branching** to create new branches for preview environments. + 8. Click **Add Integration and Return to Vercel**. ![Vercel Integration Page](/media/tidb-cloud/vercel/integration-link-cluster-page.png) @@ -100,15 +108,16 @@ The detailed steps are as follows: TIDB_PORT TIDB_USER TIDB_PASSWORD + TIDB_DATABASE ``` - For TiDB Dedicated clusters, the root CA is set in this variable: + **Prisma** ``` - TIDB_SSL_CA + DATABASE_URL ``` - **Prisma** + **TiDB Cloud Serverless Driver** ``` DATABASE_URL @@ -145,24 +154,79 @@ The detailed steps are as follows:
    +### Configure connections + +If you have installed [TiDB Cloud Vercel integration](https://vercel.com/integrations/tidb-cloud), you can add or remove connections inside the integration. + +1. In your Vercel dashboard, click **Integrations**. +2. Click **Manage** in the TiDB Cloud entry. +3. Click **Configure**. +4. Click **Add Link** or **Remove** to add or remove connections. + + ![Vercel Integration Configuration Page](/media/tidb-cloud/vercel/integration-vercel-configuration-page.png) + + When you remove a connection, the environment variables set by the integration workflow are removed from the Vercel project, too. However, this action does not affect the data of the TiDB Cloud Serverless cluster. + +### Connect with TiDB Cloud Serverless branching + +Vercel's [Preview Deployments](https://vercel.com/docs/deployments/preview-deployments) feature allows you to preview changes to your app in a live deployment without merging those changes to your Git project's production branch. With [TiDB Cloud Serverless Branching](/tidb-cloud/branch-overview.md), you can create a new instance for each branch of your Vercel project. This allows you to preview app changes in a live deployment without affecting your production data. + +> **Note:** +> +> Currently, TiDB Cloud Serverless branching only supports [Vercel projects associated with GitHub repositories](https://vercel.com/docs/deployments/git/vercel-for-github). + +To enable TiDB Cloud Serverless Branching, you need to ensure the following in the [TiDB Cloud Vercel integration workflow](#integration-workflow): + +1. Select **Cluster** as your connection type. +2. Enable **Branching** to create new branches for preview environments. + +After you push changes to the Git repository, Vercel will trigger a preview deployment. TiDB Cloud integration will automatically create a TiDB Cloud Serverless branch for the Git branch and set environment variables. The detailed steps are as follows: + +1. Create a new branch in your Git repository. + + ```shell + cd tidb-prisma-vercel-demo1 + git checkout -b new-branch + ``` + +2. Add some changes and push the changes to the remote repository. +3. Vercel will trigger a preview deployment for the new branch. + + ![Vercel Preview_Deployment](/media/tidb-cloud/vercel/vercel-preview-deployment.png) + + 1. During the deployment, TiDB Cloud integration will automatically create a TiDB Cloud Serverless branch with the same name as the Git branch. If the TiDB Cloud Serverless branch already exists, TiDB Cloud integration will skip this step. + + ![TiDB_Cloud_Branch_Check](/media/tidb-cloud/vercel/tidbcloud-branch-check.png) + + 2. After the TiDB Cloud Serverless branch is ready, TiDB Cloud integration will set environment variables in the preview deployment for the Vercel project. + + ![Preview_Envs](/media/tidb-cloud/vercel/preview-envs.png) + + 3. TiDB Cloud integration will also register a blocking check to wait for the TiDB Cloud Serverless branch to be ready. You can rerun the check manually. +4. After the check is passed, you can visit the preview deployment to see the changes. + +> **Note:** +> +> Due to a limitation of Vercel deployment workflow, the environment variable cannot be ensured to be set in the deployment. In this case, you need to redeploy the deployment. + +> **Note:** +> +> For each organization in TiDB Cloud, you can create a maximum of five TiDB Cloud Serverless branches by default. To avoid exceeding the limit, you can delete the TiDB Cloud Serverless branches that are no longer needed. For more information, see [Manage TiDB Cloud Serverless branches](/tidb-cloud/branch-manage.md). + ## Connect via manually setting environment variables -
    +
    1. Get the connection information of your TiDB cluster. You can get the connection information from the connection dialog of your cluster. To open the dialog, go to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, click the name of your target cluster to go to its overview page, and then click **Connect** in the upper-right corner. - > **Note:** - > - > For TiDB Dedicated clusters, make sure that you have set the **Allow Access from Anywhere** traffic filter in this step. - 2. Go to your Vercel dashboard > Vercel project > **Settings** > **Environment Variables**, and then [declare each environment variable value](https://vercel.com/docs/concepts/projects/environment-variables#declare-an-environment-variable) according to the connection information of your TiDB cluster. ![Vercel Environment Variables](/media/tidb-cloud/vercel/integration-vercel-environment-variables.png) -Here we use a Prisma application as an example. The following is a datasource setting in the Prisma schema file for a TiDB Serverless cluster: +Here we use a Prisma application as an example. The following is a datasource setting in the Prisma schema file for a TiDB Cloud Serverless cluster: ``` datasource db { @@ -200,16 +264,3 @@ You can get the information of ``, ``, ``, ``, a
    - -## Configure connections - -If you have installed [TiDB Cloud Vercel integration](https://vercel.com/integrations/tidb-cloud), you can add or remove connections inside the integration. - -1. In your Vercel dashboard, click **Integrations**. -2. Click **Manage** in the TiDB Cloud entry. -3. Click **Configure**. -4. Click **Add Link** or **Remove** to add or remove connections. - - ![Vercel Integration Configuration Page](/media/tidb-cloud/vercel/integration-vercel-configuration-page.png) - - When you remove a connection, environment variables set by the integration workflow are removed from the Vercel project either. The traffic filter and the data of the TiDB Cloud cluster are not affected. diff --git a/tidb-cloud/integrate-tidbcloud-with-zapier.md b/tidb-cloud/integrate-tidbcloud-with-zapier.md index 57b107dc1548d..b31a1b17d0e94 100644 --- a/tidb-cloud/integrate-tidbcloud-with-zapier.md +++ b/tidb-cloud/integrate-tidbcloud-with-zapier.md @@ -27,7 +27,7 @@ Before you start, you need: - A [Zapier account](https://zapier.com/app/login). - A [GitHub account](https://github.com/login). -- A [TiDB Cloud account](https://tidbcloud.com/signup) and a TiDB Serverless cluster on TiDB Cloud. For more details, see [TiDB Cloud Quick Start](https://docs.pingcap.com/tidbcloud/tidb-cloud-quickstart#step-1-create-a-tidb-cluster). +- A [TiDB Cloud account](https://tidbcloud.com/signup) and a TiDB Cloud Serverless cluster on TiDB Cloud. For more details, see [TiDB Cloud Quick Start](https://docs.pingcap.com/tidbcloud/tidb-cloud-quickstart#step-1-create-a-tidb-cluster). ### Step 1: Get the template @@ -160,16 +160,16 @@ The following table lists the actions supported by TiDB Cloud App. Note that som | Action | Description | Resource | |---|---|---| -| Find Cluster | Finds an existing TiDB Serverless or TiDB Dedicated cluster. | None | -| Create Cluster | Creates a new cluster. Only supports creating a TiDB Serverless cluster. | None | -| Find Database | Finds an existing database. | A TiDB Serverless cluster | -| Create Database | Creates a new database. | A TiDB Serverless cluster | -| Find Table | Finds an existing Table. | A TiDB Serverless cluster and a database | -| Create Table | Creates a new table. | A TiDB Serverless cluster and a database | -| Create Row | Creates a new row. | A TiDB Serverless cluster, a database, and a table | -| Update Row | Updates an existing row. | A TiDB Serverless cluster, a database, and a table | -| Find Row | Finds a row in a table via a lookup column. | A TiDB Serverless cluster, a database, and a table | -| Find Row (Custom Query) | Finds a row in a table via a custom query the you provide. | A TiDB Serverless cluster, a database, and a table | +| Find Cluster | Finds an existing TiDB Cloud Serverless or TiDB Cloud Dedicated cluster. | None | +| Create Cluster | Creates a new cluster. Only supports creating a TiDB Cloud Serverless cluster. | None | +| Find Database | Finds an existing database. | A TiDB Cloud Serverless cluster | +| Create Database | Creates a new database. | A TiDB Cloud Serverless cluster | +| Find Table | Finds an existing Table. | A TiDB Cloud Serverless cluster and a database | +| Create Table | Creates a new table. | A TiDB Cloud Serverless cluster and a database | +| Create Row | Creates a new row. | A TiDB Cloud Serverless cluster, a database, and a table | +| Update Row | Updates an existing row. | A TiDB Cloud Serverless cluster, a database, and a table | +| Find Row | Finds a row in a table via a lookup column. | A TiDB Cloud Serverless cluster, a database, and a table | +| Find Row (Custom Query) | Finds a row in a table via a custom query the you provide. | A TiDB Cloud Serverless cluster, a database, and a table | ## TiDB Cloud App templates diff --git a/tidb-cloud/limitations-and-quotas.md b/tidb-cloud/limitations-and-quotas.md index 77956a759c0e3..04b52ccaff92b 100644 --- a/tidb-cloud/limitations-and-quotas.md +++ b/tidb-cloud/limitations-and-quotas.md @@ -1,11 +1,11 @@ --- -title: TiDB Dedicated Limitations and Quotas +title: TiDB Cloud Dedicated Limitations and Quotas summary: Learn the limitations and quotas in TiDB Cloud. --- -# TiDB Dedicated Limitations and Quotas +# TiDB Cloud Dedicated Limitations and Quotas -TiDB Cloud limits how many of each kind of component you can create in a [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) cluster, and the common usage limitations of TiDB. In addition, there are some organization-level quotas to limit the amount of resources created by users to prevent from creating more resources than you actually need. These tables outline limits and quotas. +TiDB Cloud limits how many of each kind of component you can create in a [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) cluster, and the common usage limitations of TiDB. In addition, there are some organization-level quotas to limit the amount of resources created by users to prevent from creating more resources than you actually need. These tables outline limits and quotas. > **Note:** > @@ -15,7 +15,7 @@ TiDB Cloud limits how many of each kind of component you can create in a [TiDB D | Component | Limit | |:-|:-| -| Number of data replicas | 3 | +| Number of copies for each [data region](/tidb-cloud/tidb-cloud-glossary.md#region) | 3 | | Number of Availability Zones for a cross-zone deployment | 3 | > **Note:** @@ -29,3 +29,7 @@ TiDB Cloud limits how many of each kind of component you can create in a [TiDB D | Maximum number of total TiDB nodes for all clusters in your organization | 10 | | Maximum number of total TiKV nodes for all clusters in your organization | 15 | | Maximum number of total TiFlash nodes for all clusters in your organization | 5 | + +> **Note:** +> +> If any of these limits or quotas present a problem for your organization, please contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). diff --git a/tidb-cloud/limited-sql-features.md b/tidb-cloud/limited-sql-features.md index f330c4bc74465..305e5802b5e1a 100644 --- a/tidb-cloud/limited-sql-features.md +++ b/tidb-cloud/limited-sql-features.md @@ -5,13 +5,13 @@ summary: Learn about the limited SQL features on TiDB Cloud. # Limited SQL features on TiDB Cloud -TiDB Cloud works with almost all workloads that TiDB supports, but there are some feature differences between TiDB Self-Hosted and TiDB Dedicated/Serverless. This document describes the limitations of SQL features on TiDB Cloud. We are constantly filling in the feature gaps between TiDB Self-Hosted and TiDB Dedicated/Serverless. If you require these features or capabilities in the gap, [contact us](/tidb-cloud/tidb-cloud-support.md) for a feature request. +TiDB Cloud works with almost all workloads that TiDB supports, but there are some feature differences between TiDB Self-Managed and TiDB Cloud Dedicated/Serverless. This document describes the limitations of SQL features on TiDB Cloud. We are constantly filling in the feature gaps between TiDB Self-Managed and TiDB Cloud Dedicated/Serverless. If you require these features or capabilities in the gap, [contact us](/tidb-cloud/tidb-cloud-support.md) for a feature request. ## Statements ### Placement and range management -| Statement | TiDB Dedicated | TiDB Serverless | +| Statement | TiDB Cloud Dedicated | TiDB Cloud Serverless | |:-|:-|:-| | `ALTER PLACEMENT POLICY` | Supported | Not supported [^1] | | `CREATE PLACEMENT POLICY` | Supported | Not supported [^1] | @@ -25,7 +25,7 @@ TiDB Cloud works with almost all workloads that TiDB supports, but there are som ### Resource groups -| Statement | TiDB Dedicated | TiDB Serverless | +| Statement | TiDB Cloud Dedicated | TiDB Cloud Serverless | |:-|:-|:-| | `ALTER RESOURCE GROUP` | Supported | Not supported [^2] | | `CALIBRATE RESOURCE` | Not supported | Not supported [^2] | @@ -36,22 +36,22 @@ TiDB Cloud works with almost all workloads that TiDB supports, but there are som ### Others -| Statement | TiDB Dedicated | TiDB Serverless | +| Statement | TiDB Cloud Dedicated | TiDB Cloud Serverless | |:-|:-|:-| | `BACKUP` | Supported | Not supported [^3] | | `SHOW BACKUPS` | Supported | Not supported [^3] | | `RESTORE` | Supported | Not supported [^3] | | `SHOW RESTORES` | Supported | Not supported [^3] | -| `ADMIN RESET TELEMETRY_ID` | Supported | Telemetry is not supported on TiDB Serverless. | +| `ADMIN RESET TELEMETRY_ID` | Supported | Telemetry is not supported on TiDB Cloud Serverless. | | `ADMIN SHOW TELEMETRY` | Not supported [^4] | Not supported [^4] | | `ADMIN SHOW SLOW` | Supported | Not supported [^5] | | `ADMIN PLUGINS ENABLE` | Supported | Not supported [^8] | | `ADMIN PLUGINS DISABLE` | Supported | Not supported [^8] | -| `ALTER INSTANCE RELOAD TLS` | Supported | TiDB Serverless automatically refreshes the TLS certificate. | +| `ALTER INSTANCE RELOAD TLS` | Supported | TiDB Cloud Serverless automatically refreshes the TLS certificate. | | `LOAD DATA INFILE` | Only supports `LOAD DATA LOCAL INFILE` | Only supports `LOAD DATA LOCAL INFILE` | | `CHANGE DRAINER` | Not supported [^7] | Not supported [^7] | | `CHANGE PUMP` | Not supported [^7] | Not supported [^7] | -| `FLASHBACK CLUSTER TO TIMESTAMP` | Supported | Not supported [^3] | +| `FLASHBACK CLUSTER` | Supported | Not supported [^3] | | `LOAD STATS` | Supported | Not supported | | `SELECT ... INTO OUTFILE` | Not supported [^4] | Not supported [^4] | | `SET CONFIG` | Not supported [^4] | Not supported [^4] | @@ -60,17 +60,17 @@ TiDB Cloud works with almost all workloads that TiDB supports, but there are som | `SHOW PLUGINS` | Supported | Not supported [^8] | | `SHOW PUMP STATUS` | Not supported [^7] | Not supported [^7] | | `SHUTDOWN` | Not supported [^4] | Not supported [^4] | -| `CREATE TABLE ... AUTO_ID_CACHE` | Supported | Not supported [^12] | +| `PLAN REPLAYER` | Supported | Supported in a different way[^11] | ## Functions and operators -| Function and operator | TiDB Dedicated | TiDB Serverless | +| Function and operator | TiDB Cloud Dedicated | TiDB Cloud Serverless | |:-|:-|:-| | `SLEEP` | No Limitation | The [`SLEEP()` function](https://docs.pingcap.com/tidbcloud/miscellaneous-functions) has a limitation wherein it can only support a maximum sleep time of 300 seconds.| ## System tables -| Database | Table | TiDB Dedicated | TiDB Serverless | +| Database | Table | TiDB Cloud Dedicated | TiDB Cloud Serverless | |:-|:-|:-|:-| | `information_schema` | `ATTRIBUTES` | Supported | Not supported [^1] | | `information_schema` | `CLUSTER_CONFIG` | Not supported [^4] | Not supported [^4] | @@ -98,8 +98,6 @@ TiDB Cloud works with almost all workloads that TiDB supports, but there are som | `information_schema` | `TIDB_HOT_REGIONS` | Not supported [^4] | Not supported [^4] | | `information_schema` | `TIDB_HOT_REGIONS_HISTORY` | Supported | Not supported [^1] | | `information_schema` | `TIDB_SERVERS_INFO` | Supported | Not supported [^1] | -| `information_schema` | `TIFLASH_SEGMENTS` | Supported | Not supported [^1] | -| `information_schema` | `TIFLASH_TABLES` | Supported | Not supported [^1] | | `information_schema` | `TIKV_REGION_PEERS` | Supported | Not supported [^1] | | `information_schema` | `TIKV_REGION_STATUS` | Supported | Not supported [^1] | | `information_schema` | `TIKV_STORE_STATUS` | Supported | Not supported [^1] | @@ -121,131 +119,125 @@ TiDB Cloud works with almost all workloads that TiDB supports, but there are som | `mysql` | `gc_delete_range_done` | Not supported [^4] | Not supported [^4] | | `mysql` | `opt_rule_blacklist` | Not supported [^4] | Not supported [^4] | | `mysql` | `tidb` | Not supported [^4] | Not supported [^4] | -| `mysql` | `tidb_ttl_job_history` | Supported | Not supported [^9] | -| `mysql` | `tidb_ttl_table_status` | Supported | Not supported [^9] | -| `mysql` | `tidb_ttl_task` | Supported | Not supported [^9] | ## System variables -| Variable | TiDB Dedicated | TiDB Serverless | +| Variable | TiDB Cloud Dedicated | TiDB Cloud Serverless | |:-|:-|:-| | `datadir` | No limitation | Not supported [^1] | -| `interactive_timeout` | No limitation | Read-only [^11] | -| `max_allowed_packet` | No limitation | Read-only [^11] | +| `interactive_timeout` | No limitation | Read-only [^10] | +| `max_allowed_packet` | No limitation | Read-only [^10] | | `plugin_dir` | No limitation | Not supported [^8] | | `plugin_load` | No limitation | Not supported [^8] | -| `require_secure_transport` | Not supported [^13] | Read-only [^11] | -| `skip_name_resolve` | No limitation | Read-only [^11] | -| `sql_log_bin` | No limitation | Read-only [^11] | -| `tidb_cdc_write_source` | No limitation | Read-only [^11] | +| `require_secure_transport` | Not supported [^12] | Read-only [^10] | +| `skip_name_resolve` | No limitation | Read-only [^10] | +| `sql_log_bin` | No limitation | Read-only [^10] | +| `tidb_cdc_write_source` | No limitation | Read-only [^10] | | `tidb_check_mb4_value_in_utf8` | Not supported [^4] | Not supported [^4] | | `tidb_config` | Not supported [^4] | Not supported [^4] | -| `tidb_ddl_disk_quota` | No limitation | Read-only [^11] | -| `tidb_ddl_enable_fast_reorg` | No limitation | Read-only [^11] | -| `tidb_ddl_error_count_limit` | No limitation | Read-only [^11] | -| `tidb_ddl_flashback_concurrency` | No limitation | Read-only [^11] | -| `tidb_ddl_reorg_batch_size` | No limitation | Read-only [^11] | -| `tidb_ddl_reorg_priority` | No limitation | Read-only [^11] | -| `tidb_ddl_reorg_worker_cnt` | No limitation | Read-only [^11] | -| `tidb_enable_1pc` | No limitation | Read-only [^11] | -| `tidb_enable_async_commit` | No limitation | Read-only [^11] | -| `tidb_enable_auto_analyze` | No limitation | Read-only [^11] | +| `tidb_ddl_disk_quota` | No limitation | Read-only [^10] | +| `tidb_ddl_enable_fast_reorg` | No limitation | Read-only [^10] | +| `tidb_ddl_error_count_limit` | No limitation | Read-only [^10] | +| `tidb_ddl_flashback_concurrency` | No limitation | Read-only [^10] | +| `tidb_ddl_reorg_batch_size` | No limitation | Read-only [^10] | +| `tidb_ddl_reorg_priority` | No limitation | Read-only [^10] | +| `tidb_ddl_reorg_worker_cnt` | No limitation | Read-only [^10] | +| `tidb_enable_1pc` | No limitation | Read-only [^10] | +| `tidb_enable_async_commit` | No limitation | Read-only [^10] | +| `tidb_enable_auto_analyze` | No limitation | Read-only [^10] | | `tidb_enable_collect_execution_info` | Not supported [^4] | Not supported [^4] | -| `tidb_enable_ddl` | No limitation | Read-only [^11] | -| `tidb_enable_gc_aware_memory_track` | No limitation | Read-only [^11] | -| `tidb_enable_gogc_tuner` | No limitation | Read-only [^11] | -| `tidb_enable_local_txn` | No limitation | Read-only [^11] | -| `tidb_enable_resource_control` | No limitation | Read-only [^11] | +| `tidb_enable_ddl` | No limitation | Read-only [^10] | +| `tidb_enable_gc_aware_memory_track` | No limitation | Read-only [^10] | +| `tidb_enable_gogc_tuner` | No limitation | Read-only [^10] | +| `tidb_enable_local_txn` | No limitation | Read-only [^10] | +| `tidb_enable_resource_control` | No limitation | Read-only [^10] | | `tidb_enable_slow_log` | Not supported [^4] | Not supported [^4] | -| `tidb_enable_stmt_summary` | No limitation | Read-only [^11] | +| `tidb_enable_stmt_summary` | No limitation | Read-only [^10] | | `tidb_enable_telemetry` | Not supported [^4] | Not supported [^4] | -| `tidb_enable_top_sql` | No limitation | Read-only [^11] | -| `tidb_enable_tso_follower_proxy` | No limitation | Read-only [^11] | +| `tidb_enable_top_sql` | No limitation | Read-only [^10] | +| `tidb_enable_tso_follower_proxy` | No limitation | Read-only [^10] | | `tidb_expensive_query_time_threshold` | Not supported [^4] | Not supported [^4] | | `tidb_force_priority` | Not supported [^4] | Not supported [^4] | -| `tidb_gc_concurrency` | No limitation | Read-only [^11] | -| `tidb_gc_enable` | No limitation | Read-only [^11] | -| `tidb_gc_life_time` | No limitation | Read-only [^11] | -| `tidb_gc_max_wait_time` | No limitation | Read-only [^11] | -| `tidb_gc_run_interval` | No limitation | Read-only [^11] | -| `tidb_gc_scan_lock_mode` | No limitation | Read-only [^11] | +| `tidb_gc_concurrency` | No limitation | Read-only [^10] | +| `tidb_gc_enable` | No limitation | Read-only [^10] | +| `tidb_gc_life_time` | No limitation | Read-only [^10] | +| `tidb_gc_max_wait_time` | No limitation | Read-only [^10] | +| `tidb_gc_run_interval` | No limitation | Read-only [^10] | +| `tidb_gc_scan_lock_mode` | No limitation | Read-only [^10] | | `tidb_general_log` | Not supported [^4] | Not supported [^4] | -| `tidb_generate_binary_plan` | No limitation | Read-only [^11] | -| `tidb_gogc_tuner_threshold` | No limitation | Read-only [^11] | -| `tidb_guarantee_linearizability` | No limitation | Read-only [^11] | -| `tidb_isolation_read_engines` | No limitation | Read-only [^11] | -| `tidb_log_file_max_days` | No limitation | Read-only [^11] | +| `tidb_generate_binary_plan` | No limitation | Read-only [^10] | +| `tidb_gogc_tuner_threshold` | No limitation | Read-only [^10] | +| `tidb_guarantee_linearizability` | No limitation | Read-only [^10] | +| `tidb_isolation_read_engines` | No limitation | Read-only [^10] | +| `tidb_log_file_max_days` | No limitation | Read-only [^10] | | `tidb_memory_usage_alarm_ratio` | Not supported [^4] | Not supported [^4] | | `tidb_metric_query_range_duration` | Not supported [^4] | Not supported [^4] | | `tidb_metric_query_step` | Not supported [^4] | Not supported [^4] | | `tidb_opt_write_row_id` | Not supported [^4] | Not supported [^4] | -| `tidb_placement_mode` | No limitation | Read-only [^11] | +| `tidb_placement_mode` | No limitation | Read-only [^10] | | `tidb_pprof_sql_cpu` | Not supported [^4] | Not supported [^4] | | `tidb_record_plan_in_slow_log` | Not supported [^4] | Not supported [^4] | | `tidb_redact_log` | Not supported [^4] | Not supported [^4] | | `tidb_restricted_read_only` | Not supported [^4] | Not supported [^4] | | `tidb_row_format_version` | Not supported [^4] | Not supported [^4] | -| `tidb_scatter_region` | No limitation | Read-only [^11] | -| `tidb_server_memory_limit` | No limitation | Read-only [^11] | -| `tidb_server_memory_limit_gc_trigger` | No limitation | Read-only [^11] | -| `tidb_server_memory_limit_sess_min_size` | No limitation | Read-only [^11] | -| `tidb_simplified_metrics` | No limitation | Read-only [^11] | +| `tidb_scatter_region` | No limitation | Read-only [^10] | +| `tidb_server_memory_limit` | No limitation | Read-only [^10] | +| `tidb_server_memory_limit_gc_trigger` | No limitation | Read-only [^10] | +| `tidb_server_memory_limit_sess_min_size` | No limitation | Read-only [^10] | +| `tidb_simplified_metrics` | No limitation | Read-only [^10] | | `tidb_slow_query_file` | Not supported [^4] | Not supported [^4] | | `tidb_slow_log_threshold` | Not supported [^4] | Not supported [^4] | | `tidb_slow_txn_log_threshold` | Not supported [^4] | Not supported [^4] | -| `tidb_stats_load_sync_wait` | No limitation | Read-only [^11] | -| `tidb_stmt_summary_history_size` | No limitation | Read-only [^11] | -| `tidb_stmt_summary_internal_query` | No limitation | Read-only [^11] | -| `tidb_stmt_summary_max_sql_length` | No limitation | Read-only [^11] | -| `tidb_stmt_summary_max_stmt_count` | No limitation | Read-only [^11] | -| `tidb_stmt_summary_refresh_interval` | No limitation | Read-only [^11] | -| `tidb_sysproc_scan_concurrency` | No limitation | Read-only [^11] | +| `tidb_stats_load_sync_wait` | No limitation | Read-only [^10] | +| `tidb_stmt_summary_history_size` | No limitation | Read-only [^10] | +| `tidb_stmt_summary_internal_query` | No limitation | Read-only [^10] | +| `tidb_stmt_summary_max_sql_length` | No limitation | Read-only [^10] | +| `tidb_stmt_summary_max_stmt_count` | No limitation | Read-only [^10] | +| `tidb_stmt_summary_refresh_interval` | No limitation | Read-only [^10] | +| `tidb_sysproc_scan_concurrency` | No limitation | Read-only [^10] | | `tidb_top_sql_max_meta_count` | Not supported [^4] | Not supported [^4] | | `tidb_top_sql_max_time_series_count` | Not supported [^4] | Not supported [^4] | -| `tidb_tso_client_batch_max_wait_time` | No limitation | Read-only [^11] | -| `tidb_ttl_delete_batch_size` | No limitation | Read-only [^9] | -| `tidb_ttl_delete_rate_limit` | No limitation | Read-only [^9] | -| `tidb_ttl_delete_worker_count` | No limitation | Read-only [^9] | -| `tidb_ttl_job_enable` | No limitation | Read-only [^9] | -| `tidb_ttl_job_schedule_window_end_time` | No limitation | Read-only [^9] | -| `tidb_ttl_job_schedule_window_start_time` | No limitation | Read-only [^9] | -| `tidb_ttl_running_tasks` | No limitation | Read-only [^9] | -| `tidb_ttl_scan_batch_size` | No limitation | Read-only [^9] | -| `tidb_ttl_scan_worker_count` | No limitation | Read-only [^9] | -| `tidb_txn_mode` | No limitation | Read-only [^11] | -| `tidb_wait_split_region_finish` | No limitation | Read-only [^11] | -| `tidb_wait_split_region_timeout` | No limitation | Read-only [^11] | -| `txn_scope` | No limitation | Read-only [^11] | -| `validate_password.enable` | No limitation | Always enabled [^10] | -| `validate_password.length` | No limitation | At least `8` [^10] | -| `validate_password.mixed_case_count` | No limitation | At least `1` [^10] | -| `validate_password.number_count` | No limitation | At least `1` [^10] | -| `validate_password.policy` | No limitation | Can only be `MEDIUM` or `STRONG` [^10] | -| `validate_password.special_char_count` | No limitation | At least `1` [^10] | -| `wait_timeout` | No limitation | Read-only [^11] | - -[^1]: Configuring data placement is not supported on TiDB Serverless. - -[^2]: Configuring resource groups is not supported on TiDB Serverless. - -[^3]: To perform [Back up and Restore](/tidb-cloud/backup-and-restore-serverless.md) operations on TiDB Serverless, you can use the TiDB Cloud console instead. +| `tidb_tso_client_batch_max_wait_time` | No limitation | Read-only [^10] | +| `tidb_ttl_delete_batch_size` | No limitation | Read-only [^10] | +| `tidb_ttl_delete_rate_limit` | No limitation | Read-only [^10] | +| `tidb_ttl_delete_worker_count` | No limitation | Read-only [^10] | +| `tidb_ttl_job_schedule_window_end_time` | No limitation | Read-only [^10] | +| `tidb_ttl_job_schedule_window_start_time` | No limitation | Read-only [^10] | +| `tidb_ttl_running_tasks` | No limitation | Read-only [^10] | +| `tidb_ttl_scan_batch_size` | No limitation | Read-only [^10] | +| `tidb_ttl_scan_worker_count` | No limitation | Read-only [^10] | +| `tidb_txn_mode` | No limitation | Read-only [^10] | +| `tidb_wait_split_region_finish` | No limitation | Read-only [^10] | +| `tidb_wait_split_region_timeout` | No limitation | Read-only [^10] | +| `txn_scope` | No limitation | Read-only [^10] | +| `validate_password.enable` | No limitation | Always enabled [^9] | +| `validate_password.length` | No limitation | At least `8` [^9] | +| `validate_password.mixed_case_count` | No limitation | At least `1` [^9] | +| `validate_password.number_count` | No limitation | At least `1` [^9] | +| `validate_password.policy` | No limitation | Can only be `MEDIUM` or `STRONG` [^9] | +| `validate_password.special_char_count` | No limitation | At least `1` [^9] | +| `wait_timeout` | No limitation | Read-only [^10] | + +[^1]: Configuring data placement is not supported on TiDB Cloud Serverless. + +[^2]: Configuring resource groups is not supported on TiDB Cloud Serverless. + +[^3]: To perform [Back up and Restore](/tidb-cloud/backup-and-restore-serverless.md) operations on TiDB Cloud Serverless, you can use the TiDB Cloud console instead. [^4]: The feature is unavailable in [Security Enhanced Mode (SEM)](/system-variables.md#tidb_enable_enhanced_security). -[^5]: To track [Slow Query](/tidb-cloud/tune-performance.md#slow-query) on TiDB Serverless, you can use the TiDB Cloud console instead. +[^5]: To track [Slow Query](/tidb-cloud/tune-performance.md#slow-query) on TiDB Cloud Serverless, you can use the TiDB Cloud console instead. -[^6]: To perform [Statement Analysis](/tidb-cloud/tune-performance.md#statement-analysis) on TiDB Serverless, you can use the TiDB Cloud console instead. +[^6]: To perform [Statement Analysis](/tidb-cloud/tune-performance.md#statement-analysis) on TiDB Cloud Serverless, you can use the TiDB Cloud console instead. [^7]: Drainer and Pump are not supported on TiDB Cloud. -[^8]: Plugin is not supported on TiDB Serverless. +[^8]: Plugin is not supported on TiDB Cloud Serverless. -[^9]: [Time to live (TTL)](/time-to-live.md) is currently unavailable on TiDB Serverless. +[^9]: TiDB Cloud Serverless enforces strong password policy. -[^10]: TiDB Serverless enforces strong password policy. +[^10]: The variable is read-only on TiDB Cloud Serverless. -[^11]: The variable is read-only on TiDB Serverless. +[^11]: TiDB Cloud Serverless does not support downloading the file exported by `PLAN REPLAYER` through `${tidb-server-status-port}` as in the [example](https://docs.pingcap.com/tidb/stable/sql-plan-replayer#examples-of-exporting-cluster-information). Instead, TiDB Cloud Serverless generates a [presigned URL](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ShareObjectPreSignedURL.html) for you to download the file. Note that this URL remains valid for 10 hours after generation. -[^12]: Customizing cache size using [`AUTO_ID_CACHE`](/auto-increment.md#cache-size-control) is temporarily unavailable on TiDB Serverless. - -[^13]: Not supported. Enabling `require_secure_transport` for TiDB Dedicated clusters will result in SQL client connection failures. +[^12]: Not supported. Enabling `require_secure_transport` for TiDB Cloud Dedicated clusters will result in SQL client connection failures. diff --git a/tidb-cloud/manage-serverless-spend-limit.md b/tidb-cloud/manage-serverless-spend-limit.md index 1a3e130056d03..7e79ddc8c4eff 100644 --- a/tidb-cloud/manage-serverless-spend-limit.md +++ b/tidb-cloud/manage-serverless-spend-limit.md @@ -1,34 +1,37 @@ --- -title: Manage Spending Limit for TiDB Serverless clusters -summary: Learn how to manage spending limit for your TiDB Serverless clusters. +title: Manage Spending Limit for TiDB Cloud Serverless Scalable Clusters +summary: Learn how to manage spending limit for your TiDB Cloud Serverless scalable clusters. --- -# Manage Spending Limit for TiDB Serverless Clusters +# Manage Spending Limit for TiDB Cloud Serverless Scalable Clusters > **Note:** > -> The spending limit is only applicable to TiDB Serverless clusters. +> The spending limit is only applicable to TiDB Cloud Serverless [scalable clusters](/tidb-cloud/select-cluster-tier.md#scalable-cluster-plan). -Spending limit refers to the maximum amount of money that you are willing to spend on a particular workload in a month. It is a cost-control mechanism that allows you to set a budget for your TiDB Serverless clusters. +Spending limit refers to the maximum amount of money that you are willing to spend on a particular workload in a month. It is a cost-control mechanism that allows you to set a budget for your TiDB Cloud Serverless scalable clusters. -For each organization in TiDB Cloud, you can create a maximum of five TiDB Serverless clusters by default. To create more TiDB Serverless clusters, you need to add a credit card and set a spending limit for the usage. But if you delete some of your previous clusters before creating more, the new cluster can still be created without a credit card. +For each organization in TiDB Cloud, you can create a maximum of five [free clusters](/tidb-cloud/select-cluster-tier.md#free-cluster-plan) by default. To create more TiDB Cloud Serverless clusters, you need to add a credit card and create scalable clusters for the usage. But if you delete some of your previous clusters before creating more, the new cluster can still be created without a credit card. ## Usage quota -For the first five TiDB Serverless clusters in your organization, TiDB Cloud provides a free usage quota for each of them as follows: +For the first five TiDB Cloud Serverless clusters in your organization, whether they are free or scalable, TiDB Cloud provides a free usage quota for each of them as follows: - Row-based storage: 5 GiB +- Columnar storage: 5 GiB - [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit): 50 million RUs per month -Once the free quota of a cluster is reached, the read and write operations on this cluster will be throttled until you [increase the quota](#update-spending-limit) or the usage is reset upon the start of a new month. For example, when the storage of a cluster exceeds 5 GiB, the maximum size limit of a single transaction is reduced from 10 MiB to 1 MiB. +Once a cluster reaches its usage quota, it immediately denies any new connection attempts until you [increase the quota](#update-spending-limit) or the usage is reset upon the start of a new month. Existing connections established before reaching the quota will remain active but will experience throttling. For example, when the row-based storage of a cluster exceeds 5 GiB for a free cluster, the cluster automatically restricts any new connection attempts. -To learn more about the RU consumption of different resources (including read, write, SQL CPU, and network egress), the pricing details, and the throttled information, see [TiDB Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details). +To learn more about the RU consumption of different resources (including read, write, SQL CPU, and network egress), the pricing details, and the throttled information, see [TiDB Cloud Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details). -If you want to create a TiDB Serverless cluster with an additional quota, you can edit the spending limit on the cluster creation page. For more information, see [Create a TiDB Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md). +If you want to create a TiDB Cloud Serverless cluster with an additional quota, you can edit the spending limit on the cluster creation page. For more information, see [Create a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md). ## Update spending limit -For an existing TiDB Serverless cluster, you can increase the usage quota by updating the spending limit as follows: +For a TiDB Cloud Serverless free cluster, you can increase the usage quota by upgrading it to a scalable cluster. For an existing scalable cluster, you can adjust the monthly spending limit directly. + +To update the spending limit for a TiDB Cloud Serverless cluster, perform the following steps: 1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, click the name of your target cluster to go to its overview page. @@ -36,9 +39,9 @@ For an existing TiDB Serverless cluster, you can increase the usage quota by upd > > If you have multiple projects, you can click in the lower-left corner and switch to another project. -2. In the **Usage This Month** area, click **Get more usage quota**. +2. In the **Usage This Month** area, click **Upgrade to Scalable Cluster**. - If you have previously updated the spending limit for a cluster and want to increase it further, click **Edit**. + To adjust the spending limit for an existing scalable cluster, click **Edit**. 3. Edit the monthly spending limit as needed. If you have not added a payment method, you will need to add a credit card after editing the limit. -4. Click **Update Spending Limit**. +4. Click **Update Cluster Plan**. diff --git a/tidb-cloud/manage-user-access.md b/tidb-cloud/manage-user-access.md index d8d6c57a92631..eed166fc4f77f 100644 --- a/tidb-cloud/manage-user-access.md +++ b/tidb-cloud/manage-user-access.md @@ -7,7 +7,7 @@ summary: Learn how to manage identity access in TiDB Cloud. This document describes how to manage access to organizations, projects, roles, and user profiles in TiDB Cloud. -Before accessing TiDB Cloud, [create a TiDB cloud account](https://tidbcloud.com/free-trial). You can either sign up with email and password so that you can [manage your password using TiDB Cloud](/tidb-cloud/tidb-cloud-password-authentication.md), or choose your Google, GitHub, or Microsoft account for single sign-on (SSO) to TiDB Cloud. +Before accessing TiDB Cloud, [create a TiDB Cloud account](https://tidbcloud.com/free-trial). You can either sign up with email and password so that you can [manage your password using TiDB Cloud](/tidb-cloud/tidb-cloud-password-authentication.md), or choose your Google, GitHub, or Microsoft account for single sign-on (SSO) to TiDB Cloud. ## Organizations and projects @@ -78,13 +78,13 @@ At the organization level, TiDB Cloud defines four roles, in which `Organization | Invite users to or remove users from an organization, and edit organization roles of users. | ✅ | ❌ | ❌ | ❌ | | All the permissions of `Project Owner` for all projects in the organization. | ✅ | ❌ | ❌ | ❌ | | Create projects with Customer-Managed Encryption Key (CMEK) enabled | ✅ | ❌ | ❌ | ❌ | -| View bills and edit payment information for the organization. | ✅ | ✅ | ❌ | ❌ | +| View bills, use [cost explorer](/tidb-cloud/tidb-cloud-billing.md#cost-explorer), and edit payment information for the organization. | ✅ | ✅ | ❌ | ❌ | | Manage TiDB Cloud [console audit logging](/tidb-cloud/tidb-cloud-console-auditing.md) for the organization. | ✅ | ❌ | ✅ | ❌ | | View users in the organization and projects in which the member belong to. | ✅ | ✅ | ✅ | ✅ | > **Note:** > -> The `Organization Console Audit Admin` role is only visible upon request. It is recommended that you use the `Organization Owner` role for [console audit logging](/tidb-cloud/tidb-cloud-console-auditing.md). If you need to use the `Organization Console Audit Admin` role, click **?** in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com) and click **Request Support**. Then, fill in "Apply for the Organization Console Audit Admin role" in the **Description** field and click **Send**. +> The `Organization Console Audit Admin` role is used to manage audit logging in the TiDB Cloud console, instead of database audit logging. To manage database auditing, use the `Project Owner` role at the project level. ### Project roles @@ -101,14 +101,15 @@ At the project level, TiDB Cloud defines three roles, in which `Project Owner` c | Manage project settings | ✅ | ❌ | ❌ | | Invite users to or remove users from a project, and edit project roles of users. | ✅ | ❌ | ❌ | | Manage [database audit logging](/tidb-cloud/tidb-cloud-auditing.md) of the project. | ✅ | ❌ | ❌ | -| Manage [spending limit](/tidb-cloud/manage-serverless-spend-limit.md) for all TiDB Serverless clusters in the project. | ✅ | ❌ | ❌ | +| Manage [spending limit](/tidb-cloud/manage-serverless-spend-limit.md) for all TiDB Cloud Serverless clusters in the project. | ✅ | ❌ | ❌ | | Manage cluster operations in the project, such as cluster creation, modification, and deletion. | ✅ | ❌ | ❌ | -| Manage branches for TiDB Serverless clusters in the project, such as branch creation, connection, and deletion. | ✅ | ❌ | ❌ | +| Manage branches for TiDB Cloud Serverless clusters in the project, such as branch creation, connection, and deletion. | ✅ | ❌ | ❌ | +| Manage [recovery groups](/tidb-cloud/recovery-group-overview.md) for TiDB Cloud Dedicated clusters in the project, such as recovery group creation and deletion. | ✅ | ❌ | ❌ | | Manage cluster data such as data import, data backup and restore, and data migration. | ✅ | ✅ | ❌ | | Manage [Data Service](/tidb-cloud/data-service-overview.md) for data read-only operations such as using or creating endpoints to read data. | ✅ | ✅ | ✅ | | Manage [Data Service](/tidb-cloud/data-service-overview.md) for data read and write operations. | ✅ | ✅ | ❌ | -| View cluster data using [Chat2Query](/tidb-cloud/explore-data-with-chat2query.md). | ✅ | ✅ | ✅ | -| Modify and delete cluster data using [Chat2Query](/tidb-cloud/explore-data-with-chat2query.md). | ✅ | ✅ | ❌ | +| View cluster data using [SQL Editor](/tidb-cloud/explore-data-with-chat2query.md). | ✅ | ✅ | ✅ | +| Modify and delete cluster data using [SQL Editor](/tidb-cloud/explore-data-with-chat2query.md). | ✅ | ✅ | ❌ | | View clusters in the project, view cluster backup records, and manage [changefeeds](/tidb-cloud/changefeed-overview.md). | ✅ | ✅ | ✅ | ## Manage organization access @@ -139,11 +140,9 @@ To change the local timezone setting, take the following steps: 2. Click **Organization Settings**. The organization settings page is displayed. -3. Click the **Time Zone** tab. +3. In the **Time Zone** section, select your time zone from the drop-down list. -4. Click the drop-down list and select your time zone. - -5. Click **Save**. +4. Click **Update**. ### Invite an organization member @@ -159,7 +158,7 @@ To invite a member to an organization, take the following steps: 2. Click **Organization Settings**. The organization settings page is displayed. -3. Click the **User Management** tab, and then select **By Organization**. +3. Click the **Users** tab in the left navigation pane, and then select **By Organization**. 4. Click **Invite**. @@ -190,7 +189,7 @@ To modify the organization role of a member, take the following steps: 2. Click **Organization Settings**. The organization settings page is displayed. -3. Click the **User Management** tab, and then select **By Organization**. +3. Click the **Users** tab in the left navigation pane, and then select **By Organization**. 4. Click the role of the target member, and then modify the role. @@ -208,7 +207,7 @@ To remove a member from an organization, take the following steps: 2. Click **Organization Settings**. The organization settings page is displayed. -3. Click the **User Management** tab, and then select **By Organization**. +3. Click the **Users** tab in the left navigation pane, and then select **By Organization**. 4. Click **Delete** in the user row that you want to delete. @@ -220,7 +219,7 @@ To check which project you belong to, take the following steps: 1. Click in the lower-left corner of the TiDB Cloud console. -2. Click **Organization Settings**. The **Projects** tab is displayed by default. +2. Click **Organization Settings**, and then click the **Projects** tab in the left navigation pane. The **Projects** tab is displayed. > **Tip:** > @@ -238,7 +237,7 @@ To create a new project, take the following steps: 1. Click in the lower-left corner of the TiDB Cloud console. -2. Click **Organization Settings**. The **Projects** tab is displayed by default. +2. Click **Organization Settings**, and then click the **Projects** tab in the left navigation pane. The **Projects** tab is displayed. 3. Click **Create New Project**. @@ -254,7 +253,7 @@ To rename a project, take the following steps: 1. Click in the lower-left corner of the TiDB Cloud console. -2. Click **Organization Settings**. The **Projects** tab is displayed by default. +2. Click **Organization Settings**, and then click the **Projects** tab in the left navigation pane. The **Projects** tab is displayed. 3. In the row of your project to be renamed, click **Rename**. diff --git a/tidb-cloud/managed-service-provider-customer.md b/tidb-cloud/managed-service-provider-customer.md new file mode 100644 index 0000000000000..7fd199970f6c3 --- /dev/null +++ b/tidb-cloud/managed-service-provider-customer.md @@ -0,0 +1,43 @@ +--- +title: Managed Service Provider Customer +summary: Learn how to become a Managed Service Provider (MSP) customer. +--- + +# Managed Service Provider Customer + +A Managed Service Provider (MSP) customer is a customer who uses the TiDB Cloud Services provided by the Managed Service Provider. + +Compared to the direct TiDB Cloud customer, there are several differences for sign-up and invoice payment: + +- The MSP customer needs to sign up for a TiDB Cloud account from the dedicated sign-up page provided by the MSP. +- The MSP customer pays invoices through the MSP channel, instead of paying directly to PingCAP. + +Other daily operations on the TiDB Cloud console are the same for both direct TiDB Cloud customers and MSP customers. + +This document describes how to become an MSP customer and how to check history and future bills for MSP customers. + +## Create a new MSP customer account + +To create a new MSP customer account, visit the MSP dedicated sign-up page. Each MSP has a unique dedicated sign-up page. Contact your MSP to get the URL of the sign-up page. + +## Migrate from a direct TiDB Cloud account to an MSP customer account + +> **Tip:** +> +> Direct customers are the end customers who purchase TiDB Cloud and pay invoices directly from PingCAP. + +If you are currently a direct customer with a TiDB Cloud account, you can ask your MSP to migrate your account to an MSP customer account. + +The migration will take effect on the first day of a future month. Discuss with your MSP to determine the specific effective date. + +On the effective day of migration, you will receive an email notification. + +## Check your future bills + +Once you successfully become an MSP customer, you will pay invoices through your MSP. Ask your MSP where you can check your bills and invoices. + +PingCAP does not send any bills and invoices to MSP customers. + +## Check your history bills + +If you have migrated from a direct TiDB Cloud account to an MSP customer account, you can view your history bills prior to the migration by visiting **Billing** > **Bills** > **History** in the TiDB Cloud console. diff --git a/tidb-cloud/migrate-from-mysql-using-aws-dms.md b/tidb-cloud/migrate-from-mysql-using-aws-dms.md index 70e14bce6bf53..4a7fc78e0ddd7 100644 --- a/tidb-cloud/migrate-from-mysql-using-aws-dms.md +++ b/tidb-cloud/migrate-from-mysql-using-aws-dms.md @@ -11,7 +11,7 @@ AWS DMS is a cloud service that makes it easy to migrate relational databases, d This document uses Amazon RDS as an example to show how to migrate data to TiDB Cloud using AWS DMS. The procedure also applies to migrating data from self-hosted MySQL databases or Amazon Aurora to TiDB Cloud. -In this example, the data source is Amazon RDS, and the data destination is a TiDB Dedicated cluster in TiDB Cloud. Both upstream and downstream databases are in the same region. +In this example, the data source is Amazon RDS, and the data destination is a TiDB Cloud Dedicated cluster in TiDB Cloud. Both upstream and downstream databases are in the same region. ## Prerequisites @@ -28,7 +28,8 @@ Before you start the migration, make sure you have read the following: ## Limitation -AWS DMS does not support replicating `DROP TABLE`. +- AWS DMS does not support replicating `DROP TABLE`. +- AWS DMS supports basic schema migration, including the creation of tables and primary keys. However, AWS DMS does not automatically create secondary indexes, foreign keys, or user accounts in TiDB Cloud. You must manually create these objects in TiDB, including tables with secondary indexes, if needed. For more information, see [Migration planning for AWS Database Migration Service](https://docs.aws.amazon.com/dms/latest/userguide/CHAP_BestPractices.html#CHAP_SettingUp.MigrationPlanning). ## Step 1. Create an AWS DMS replication instance @@ -100,7 +101,7 @@ AWS DMS does not support replicating `DROP TABLE`. 3. Under **Step 1: Create traffic filter** in the dialog, click **Edit**, enter the public and private network IP addresses that you copied from the AWS DMS console, and then click **Update Filter**. It is recommended to add the public IP address and private IP address of the AWS DMS replication instance to the TiDB cluster traffic filter at the same time. Otherwise, AWS DMS might not be able to connect to the TiDB cluster in some scenarios. -4. Click **Download TiDB cluster CA** to download the CA certificate. Under **Step 3: Connect with a SQL client** in the dialog, take a note of the `-u`, `-h`, and `-P` information in the connection string for later use. +4. Click **Download CA cert** to download the CA certificate. Under **Step 3: Connect with a SQL client** in the dialog, take a note of the `-u`, `-h`, and `-P` information in the connection string for later use. 5. Click the **VPC Peering** tab in the dialog, and then click **Add** under **Step 1: Set up VPC** to create a VPC Peering connection for the TiDB cluster and AWS DMS. @@ -180,6 +181,8 @@ If you encounter any issues or failures during the migration, you can check the ## See also +- If you want to learn more about how to connect AWS DMS to TiDB Cloud Serverless or TiDB Cloud Dedicated, see [Connect AWS DMS to TiDB Cloud clusters](/tidb-cloud/tidb-cloud-connect-aws-dms.md). + - If you want to migrate from MySQL-compatible databases, such as Aurora MySQL and Amazon Relational Database Service (RDS), to TiDB Cloud, it is recommended to use [Data Migration on TiDB Cloud](/tidb-cloud/migrate-from-mysql-using-data-migration.md). -- If you want to migrate from Amazon RDS for Oracle to TiDB Serverless Using AWS DMS, see [Migrate from Amazon RDS for Oracle to TiDB Serverless Using AWS DMS](/tidb-cloud/migrate-from-oracle-using-aws-dms.md). +- If you want to migrate from Amazon RDS for Oracle to TiDB Cloud Serverless Using AWS DMS, see [Migrate from Amazon RDS for Oracle to TiDB Cloud Serverless Using AWS DMS](/tidb-cloud/migrate-from-oracle-using-aws-dms.md). diff --git a/tidb-cloud/migrate-from-mysql-using-data-migration.md b/tidb-cloud/migrate-from-mysql-using-data-migration.md index cd9d36f220aa8..ca58ca4a5e8a0 100644 --- a/tidb-cloud/migrate-from-mysql-using-data-migration.md +++ b/tidb-cloud/migrate-from-mysql-using-data-migration.md @@ -14,35 +14,35 @@ If you want to migrate incremental data only, see [Migrate Incremental Data from ## Limitations -- The Data Migration feature is available only for **TiDB Dedicated** clusters. +### Availability -- The Data Migration feature is only available to clusters in the projects that are created in the following regions after November 9, 2022. If your **project** was created before the date or if your cluster is in another region, this feature is not available to your cluster and the **Data Migration** tab will not be displayed on the cluster overview page in the TiDB Cloud console. +- The Data Migration feature is available only for **TiDB Cloud Dedicated** clusters. - - AWS Oregon (us-west-2) - - AWS N. Virginia (us-east-1) - - AWS Singapore (ap-southeast-1) - - AWS Tokyo (ap-northeast-1) - - AWS Frankfurt (eu-central-1) - - AWS Seoul (ap-northeast-2) - - Google Cloud Oregon (us-west1) - - Google Cloud Singapore (asia-southeast1) - - Google Cloud Tokyo (asia-northeast1) +- The Data Migration feature is only available to clusters that are created in [certain regions](https://www.pingcap.com/tidb-cloud-pricing-details/#dm-cost) after November 9, 2022. If your **project** was created before the date or if your cluster is in another region, this feature is not available to your cluster and the **Data Migration** tab will not be displayed on the cluster overview page in the TiDB Cloud console. - Amazon Aurora MySQL writer instances support both existing data and incremental data migration. Amazon Aurora MySQL reader instances only support existing data migration and do not support incremental data migration. -- You can create up to 200 migration jobs for each organization. To create more migration jobs, you need to [file a support ticket](/tidb-cloud/tidb-cloud-support.md). +### Maximum number of migration jobs + +You can create up to 200 migration jobs for each organization. To create more migration jobs, you need to [file a support ticket](/tidb-cloud/tidb-cloud-support.md). + +### Filtered out and deleted databases - The system databases will be filtered out and not migrated to TiDB Cloud even if you select all of the databases to migrate. That is, `mysql`, `information_schema`, `information_schema`, and `sys` will not be migrated using this feature. +- When you delete a cluster in TiDB Cloud, all migration jobs in that cluster are automatically deleted and not recoverable. + +### Limitations of existing data migration + - During existing data migration, if the table to be migrated already exists in the target database with duplicated keys, the duplicate keys will be replaced. -- During incremental data migration, if the table to be migrated already exists in the target database with duplicated keys, an error is reported and the migration is interrupted. In this situation, you need to make sure whether the upstream data is accurate. If yes, click the "Restart" button of the migration job and the migration job will replace the downstream conflicting records with the upstream records. +- If your dataset size is smaller than 1 TiB, it is recommended that you use logical mode (the default mode). If your dataset size is larger than 1 TiB, or you want to migrate existing data faster, you can use physical mode. For more information, see [Migrate existing data and incremental data](#migrate-existing-data-and-incremental-data). -- When you delete a cluster in TiDB Cloud, all migration jobs in that cluster are automatically deleted and not recoverable. +### Limitations of incremental data migration -- During incremental replication (migrating ongoing changes to your cluster), if the migration job recovers from an abrupt error, it might open the safe mode for 60 seconds. During the safe mode, `INSERT` statements are migrated as `REPLACE`, `UPDATE` statements as `DELETE` and `REPLACE`, and then these transactions are migrated to the downstream cluster to make sure that all the data during the abrupt error has been migrated smoothly to the downstream cluster. In this scenario, for upstream tables without primary keys or not-null unique indexes, some data might be duplicated in the downstream cluster because the data might be inserted repeatedly to the downstream. +- During incremental data migration, if the table to be migrated already exists in the target database with duplicated keys, an error is reported and the migration is interrupted. In this situation, you need to make sure whether the upstream data is accurate. If yes, click the "Restart" button of the migration job and the migration job will replace the downstream conflicting records with the upstream records. -- When you use Data Migration, it is recommended to keep the size of your dataset smaller than 1 TiB. If the dataset size is larger than 1 TiB, the existing data migration will take a long time due to limited specifications. +- During incremental replication (migrating ongoing changes to your cluster), if the migration job recovers from an abrupt error, it might open the safe mode for 60 seconds. During the safe mode, `INSERT` statements are migrated as `REPLACE`, `UPDATE` statements as `DELETE` and `REPLACE`, and then these transactions are migrated to the downstream cluster to make sure that all the data during the abrupt error has been migrated smoothly to the downstream cluster. In this scenario, for upstream tables without primary keys or not-null unique indexes, some data might be duplicated in the downstream cluster because the data might be inserted repeatedly to the downstream. - In the following scenarios, if the migration job takes longer than 24 hours, do not purge binary logs in the source database to ensure that Data Migration can get consecutive binary logs for incremental replication: @@ -93,25 +93,20 @@ The username you use for the downstream TiDB Cloud cluster must have the followi | `ALTER` | Tables | | `DROP` | Databases, Tables | | `INDEX` | Tables | -| `TRUNCATE` | Tables | For example, you can execute the following `GRANT` statement to grant corresponding privileges: ```sql -GRANT CREATE,SELECT,INSERT,UPDATE,DELETE,ALTER,TRUNCATE,DROP,INDEX ON *.* TO 'your_user'@'your_IP_address_of_host' +GRANT CREATE,SELECT,INSERT,UPDATE,DELETE,ALTER,DROP,INDEX ON *.* TO 'your_user'@'your_IP_address_of_host' ``` To quickly test a migration job, you can use the `root` account of the TiDB Cloud cluster. ### Set up network connection -Before creating a migration job, set up the network connection according to your connection methods. See [Connect to Your TiDB Dedicated Cluster](/tidb-cloud/connect-to-tidb-cluster.md). +Before creating a migration job, set up the network connection according to your connection methods. See [Connect to Your TiDB Cloud Dedicated Cluster](/tidb-cloud/connect-to-tidb-cluster.md). -- If you use public IP (this is, standard connection) for network connection, make sure that the upstream database can be connected through the public network. - -- If you use AWS PrivateLink, set it up according to [Connect to TiDB Dedicated via Private Endpoint with AWS](/tidb-cloud/set-up-private-endpoint-connections.md). - -- If you use Google Cloud Private Service Connect, set it up according to [Connect to TiDB Dedicated via Private Endpoint with Google Cloud](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md). +- If you use public IP (this is, public connection) for network connection, make sure that the upstream database can be connected through the public network. - If you use AWS VPC Peering or Google Cloud VPC Network Peering, see the following instructions to configure the network. @@ -124,7 +119,7 @@ If your MySQL service is in an AWS VPC, take the following steps: 2. Modify the inbound rules of the security group that the MySQL service is associated with. - You must add [the CIDR of the region where your TiDB Cloud cluster is located](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-project-cidr) to the inbound rules. Doing so allows the traffic to flow from your TiDB cluster to the MySQL instance. + You must add [the CIDR of the region where your TiDB Cloud cluster is located](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-cidr-for-a-region) to the inbound rules. Doing so allows the traffic to flow from your TiDB cluster to the MySQL instance. 3. If the MySQL URL contains a DNS hostname, you need to allow TiDB Cloud to be able to resolve the hostname of the MySQL service. @@ -144,7 +139,7 @@ If your MySQL service is in a Google Cloud VPC, take the following steps: 3. Modify the ingress firewall rules of the VPC where MySQL is located. - You must add [the CIDR of the region where your TiDB Cloud cluster is located](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-project-cidr) to the ingress firewall rules. This allows the traffic to flow from your TiDB cluster to the MySQL endpoint. + You must add [the CIDR of the region where your TiDB Cloud cluster is located](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-cidr-for-a-region) to the ingress firewall rules. This allows the traffic to flow from your TiDB cluster to the MySQL endpoint. @@ -205,10 +200,34 @@ In the **Choose the objects to be migrated** step, you can choose existing data To migrate data to TiDB Cloud once and for all, choose both **Existing data migration** and **Incremental data migration**, which ensures data consistency between the source and target databases. +You can use **physical mode** or **logical mode** to migrate **existing data** and **incremental data**. + +- The default mode is **logical mode**. This mode exports data from upstream databases as SQL statements, and then executes them on TiDB. In this mode, the target tables before migration can be either empty or non-empty. But the performance is slower than physical mode. + +- For large datasets, it is recommended to use **physical mode**. This mode exports data from upstream databases and encodes it as KV pairs, writing directly to TiKV to achieve faster performance. This mode requires the target tables to be empty before migration. For the specification of 16 RCUs (Replication Capacity Units), the performance is about 2.5 times faster than logical mode. The performance of other specifications can increase by 20% to 50% compared with logical mode. Note that the performance data is for reference only and might vary in different scenarios. + +Physical mode is available for TiDB clusters deployed on AWS and Google Cloud. + +> **Note:** +> +> - When you use physical mode, you cannot create a second migration job or import task for the TiDB cluster before the existing data migration is completed. +> - When you use physical mode and the migration job has started, do **NOT** enable PITR (Point-in-time Recovery) or have any changefeed on the cluster. Otherwise, the migration job will be stuck. If you need to enable PITR or have any changefeed, use logical mode instead to migrate data. + +Physical mode exports the upstream data as fast as possible, so [different specifications](/tidb-cloud/tidb-cloud-billing-dm.md#specifications-for-data-migration) have different performance impacts on QPS and TPS of the upstream database during data export. The following table shows the performance regression of each specification. + +| Migration specification | Maximum export speed | Performance regression of the upstream database | +|---------|-------------|--------| +| 2 RCUs | 80.84 MiB/s | 15.6% | +| 4 RCUs | 214.2 MiB/s | 20.0% | +| 8 RCUs | 365.5 MiB/s | 28.9% | +| 16 RCUs | 424.6 MiB/s | 46.7% | + ### Migrate only existing data To migrate only existing data of the source database to TiDB Cloud, choose **Existing data migration**. +You can only use logical mode to migrate existing data. For more information, see [Migrate existing data and incremental data](#migrate-existing-data-and-incremental-data). + ### Migrate only incremental data To migrate only the incremental data of the source database to TiDB Cloud, choose **Incremental data migration**. In this case, the migration job does not migrate the existing data of the source database to TiDB Cloud, but only migrates the ongoing changes of the source database that are explicitly specified by the migration job. @@ -221,15 +240,15 @@ For detailed instructions about incremental data migration, see [Migrate Only In - If you click **All**, the migration job will migrate the existing data from the whole source database instance to TiDB Cloud and migrate ongoing changes after the full migration. Note that it happens only if you have selected the **Existing data migration** and **Incremental data migration** checkboxes in the previous step. - + - If you click **Customize** and select some databases, the migration job will migrate the existing data and migrate ongoing changes of the selected databases to TiDB Cloud. Note that it happens only if you have selected the **Existing data migration** and **Incremental data migration** checkboxes in the previous step. - + - If you click **Customize** and select some tables under a dataset name, the migration job only will migrate the existing data and migrate ongoing changes of the selected tables. Tables created afterwards in the same database will not be migrated. - + -TiDB Serverless is a fully managed, multi-tenant TiDB offering. It delivers an instant, autoscaling MySQL-compatible database and offers a generous free tier and consumption based billing once free limits are exceeded. +TiDB Cloud Serverless is a fully managed, multi-tenant TiDB offering. It delivers an instant, autoscaling MySQL-compatible database and offers a generous free tier and consumption based billing once free limits are exceeded. + +### Cluster plans + +TiDB Cloud Serverless offers two service plans to meet different user requirements. Whether you are just getting started or scaling to meet the increasing application demands, these service plans provide the flexibility and capability you need. + +#### Free cluster plan + +The free cluster plan is ideal for those who are getting started with TiDB Cloud Serverless. It provides developers and small teams with the following essential features: + +- **No cost**: This plan is completely free, with no credit card required to get started. +- **Storage**: Provides an initial 5 GiB of row-based storage and 5 GiB of columnar storage. +- **Request Units**: Includes 50 million [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit) for database operations. +- **Easy upgrade**: Offers a smooth transition to the [scalable cluster plan](#scalable-cluster-plan) as your needs grow. + +#### Scalable cluster plan + +For applications experiencing growing workloads and needing scalability in real time, the scalable cluster plan provides the flexibility and performance to keep pace with your business growth with the following features: + +- **Enhanced capabilities**: Includes all capabilities of the free cluster plan, along with the capacity to handle larger and more complex workloads, as well as advanced security features. +- **Automatic scaling**: Automatically adjusts storage and computing resources to efficiently meet changing workload demands. +- **Predictable pricing**: Although this plan requires a credit card, you are only charged for the resources you use, ensuring cost-effective scalability. ### Usage quota -For each organization in TiDB Cloud, you can create a maximum of five TiDB Serverless clusters by default. To create more TiDB Serverless clusters, you need to add a credit card and set a [spending limit](/tidb-cloud/tidb-cloud-glossary.md#spending-limit) for the usage. +For each organization in TiDB Cloud, you can create a maximum of five [free clusters](#free-cluster-plan) by default. To create more TiDB Cloud Serverless clusters, you need to add a credit card and create [scalable clusters](#scalable-cluster-plan) for the usage. -For the first five TiDB Serverless clusters in your organization, TiDB Cloud provides a free usage quota for each of them as follows: +For the first five TiDB Cloud Serverless clusters in your organization, whether they are free or scalable, TiDB Cloud provides a free usage quota for each of them as follows: - Row-based storage: 5 GiB -- [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit): 50 million RUs per month +- Columnar storage: 5 GiB +- Request Units (RUs): 50 million RUs per month A Request Unit (RU) is a unit of measure used to represent the amount of resources consumed by a single request to the database. The amount of RUs consumed by a request depends on various factors, such as the operation type or the amount of data being retrieved or modified. -Once the free quota of a cluster is reached, the read and write operations on this cluster will be throttled until you [increase the quota](/tidb-cloud/manage-serverless-spend-limit.md#update-spending-limit) or the usage is reset upon the start of a new month. For example, when the storage of a cluster exceeds 5 GiB, the maximum size limit of a single transaction is reduced from 10 MiB to 1 MiB. +Once a cluster reaches its usage quota, it immediately denies any new connection attempts until you [increase the quota](/tidb-cloud/manage-serverless-spend-limit.md#update-spending-limit) or the usage is reset upon the start of a new month. Existing connections established before reaching the quota will remain active but will experience throttling. For example, when the row-based storage of a cluster exceeds 5 GiB for a free cluster, the cluster automatically restricts any new connection attempts. -To learn more about the RU consumption of different resources (including read, write, SQL CPU, and network egress), the pricing details, and the throttled information, see [TiDB Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details). +To learn more about the RU consumption of different resources (including read, write, SQL CPU, and network egress), the pricing details, and the throttled information, see [TiDB Cloud Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details). ### User name prefix -For each TiDB Serverless cluster, TiDB Cloud generates a unique prefix to distinguish it from other clusters. +For each TiDB Cloud Serverless cluster, TiDB Cloud generates a unique prefix to distinguish it from other clusters. Whenever you use or set a database user name, you must include the prefix in the user name. For example, assume that the prefix of your cluster is `3pTAoNNegb47Uc8`. @@ -49,7 +71,7 @@ Whenever you use or set a database user name, you must include the prefix in the > **Note:** > - > TiDB Serverless requires TLS connection. To find the CA root path on your system, see [Root certificate default path](/tidb-cloud/secure-connections-to-serverless-clusters.md#root-certificate-default-path). + > TiDB Cloud Serverless requires TLS connection. To find the CA root path on your system, see [Root certificate default path](/tidb-cloud/secure-connections-to-serverless-clusters.md#root-certificate-default-path). - To create a database user: @@ -63,17 +85,17 @@ To get the prefix for your cluster, take the following steps: 2. Click the name of your target cluster to go to its overview page, and then click **Connect** in the upper-right corner. A connection dialog is displayed. 3. In the dialog, get the prefix from the connection string. -### TiDB Serverless special terms and conditions +### TiDB Cloud Serverless special terms and conditions -Some of TiDB Cloud features are partially supported or not supported on TiDB Serverless. See [TiDB Serverless Limitations](/tidb-cloud/serverless-limitations.md) for details. +Some of TiDB Cloud features are partially supported or not supported on TiDB Cloud Serverless. See [TiDB Cloud Serverless Limitations](/tidb-cloud/serverless-limitations.md) for details. -## TiDB Dedicated +## TiDB Cloud Dedicated -TiDB Dedicated is for production use with the benefits of cross-zone high availability, horizontal scaling, and [HTAP](https://en.wikipedia.org/wiki/Hybrid_transactional/analytical_processing). +TiDB Cloud Dedicated is for production use with the benefits of cross-zone high availability, horizontal scaling, and [HTAP](https://en.wikipedia.org/wiki/Hybrid_transactional/analytical_processing). -For TiDB Dedicated clusters, you can customize the cluster size of TiDB, TiKV, and TiFlash easily according to your business need. For each TiKV node and TiFlash node, the data on the node is replicated and distributed in different availability zones for [high availability](/tidb-cloud/high-availability-with-multi-az.md). +For TiDB Cloud Dedicated clusters, you can customize the cluster size of TiDB, TiKV, and TiFlash easily according to your business need. For each TiKV node and TiFlash node, the data on the node is replicated and distributed in different availability zones for [high availability](/tidb-cloud/high-availability-with-multi-az.md). -To create a TiDB Dedicated cluster, you need to [add a payment method](/tidb-cloud/tidb-cloud-billing.md#payment-method) or [apply for a Proof of Concept (PoC) trial](/tidb-cloud/tidb-cloud-poc.md). +To create a TiDB Cloud Dedicated cluster, you need to [add a payment method](/tidb-cloud/tidb-cloud-billing.md#payment-method) or [apply for a Proof of Concept (PoC) trial](/tidb-cloud/tidb-cloud-poc.md). > **Note:** > diff --git a/tidb-cloud/serverless-driver-drizzle-example.md b/tidb-cloud/serverless-driver-drizzle-example.md new file mode 100644 index 0000000000000..cae90ab0ec24b --- /dev/null +++ b/tidb-cloud/serverless-driver-drizzle-example.md @@ -0,0 +1,272 @@ +--- +title: TiDB Cloud Serverless Driver Drizzle Tutorial +summary: Learn how to use TiDB Cloud serverless driver with Drizzle. +--- + +# TiDB Cloud Serverless Driver Drizzle Tutorial + +[Drizzle ORM](https://orm.drizzle.team/) is a lightweight and performant TypeScript ORM with developer experience in mind. Starting from `drizzle-orm@0.31.2`, it supports [drizzle-orm/tidb-serverless](https://orm.drizzle.team/docs/get-started-mysql#tidb-serverless), enabling you to use Drizzle over HTTPS with [TiDB Cloud serverless driver](/tidb-cloud/serverless-driver.md). + +This tutorial describes how to use TiDB Cloud serverless driver with Drizzle in Node.js environments and edge environments. + +## Use Drizzle and TiDB Cloud serverless driver in Node.js environments + +This section describes how to use TiDB Cloud serverless driver with Drizzle in Node.js environments. + +### Before you begin + +To complete this tutorial, you need the following: + +- [Node.js](https://nodejs.org/en) >= 18.0.0. +- [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) or your preferred package manager. +- A TiDB Cloud Serverless cluster. If you don't have any, you can [create a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md). + +### Step 1. Create a project + +1. Create a project named `drizzle-node-example`: + + ```shell + mkdir drizzle-node-example + cd drizzle-node-example + ``` + +2. Install the `drizzle-orm` and `@tidbcloud/serverless` packages: + + ```shell + npm install drizzle-orm @tidbcloud/serverless + ``` + +3. In the root directory of your project, locate the `package.json` file, and then specify the ES module by adding `"type": "module"` to the file: + + ```json + { + "type": "module", + "dependencies": { + "@tidbcloud/serverless": "^0.1.1", + "drizzle-orm": "^0.31.2" + } + } + ``` + +4. In the root directory of your project, add a `tsconfig.json` file to define the TypeScript compiler options. Here is an example file: + + ```json + { + "compilerOptions": { + "module": "ES2022", + "target": "ES2022", + "moduleResolution": "node", + "strict": false, + "declaration": true, + "outDir": "dist", + "removeComments": true, + "allowJs": true, + "esModuleInterop": true, + "resolveJsonModule": true + } + } + ``` + +### Step 2. Set the environment + +1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Cloud Serverless cluster to go to its overview page. + +2. On the overview page, click **Connect** in the upper-right corner, select `Serverless Driver` in the **Connect With** drop-down box, and then click **Generate Password** to create a random password. + + > **Tip:** + > + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. + + The connection string looks like this: + + ``` + mysql://[username]:[password]@[host]/[database] + ``` + +3. Set the environment variable `DATABASE_URL` in your local environment. For example, in Linux or macOS, you can run the following command: + + ```shell + export DATABASE_URL='mysql://[username]:[password]@[host]/[database]' + ``` + +### Step 3. Use Drizzle to query data + +1. Create a table in your TiDB Cloud Serverless cluster. + + You can use [SQL Editor in the TiDB Cloud console](/tidb-cloud/explore-data-with-chat2query.md) to execute SQL statements. Here is an example: + + ```sql + CREATE TABLE `test`.`users` ( + `id` BIGINT PRIMARY KEY auto_increment, + `full_name` TEXT, + `phone` VARCHAR(256) + ); + ``` + +2. In the root directory of your project, create a file named `hello-world.ts` and add the following code: + + ```ts + import { connect } from '@tidbcloud/serverless'; + import { drizzle } from 'drizzle-orm/tidb-serverless'; + import { mysqlTable, serial, text, varchar } from 'drizzle-orm/mysql-core'; + + // Initialize + const client = connect({ url: process.env.DATABASE_URL }); + const db = drizzle(client); + + // Define schema + export const users = mysqlTable('users', { + id: serial("id").primaryKey(), + fullName: text('full_name'), + phone: varchar('phone', { length: 256 }), + }); + export type User = typeof users.$inferSelect; // return type when queried + export type NewUser = typeof users.$inferInsert; // insert type + + // Insert and select data + const user: NewUser = { fullName: 'John Doe', phone: '123-456-7890' }; + await db.insert(users).values(user) + const result: User[] = await db.select().from(users); + console.log(result); + ``` + +### Step 4. Run the Typescript code + +1. Install `ts-node` to transform TypeScript into JavaScript, and then install `@types/node` to provide TypeScript type definitions for Node.js. + + ```shell + npm install -g ts-node + npm i --save-dev @types/node + ``` + +2. Run the Typescript code with the following command: + + ```shell + ts-node --esm hello-world.ts + ``` + +## Use Drizzle and TiDB Cloud serverless driver in edge environments + +This section takes the Vercel Edge Function as an example. + +### Before you begin + +To complete this tutorial, you need the following: + +- A [Vercel](https://vercel.com/docs) account that provides edge environment. +- [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) or your preferred package manager. +- A TiDB Cloud Serverless cluster. If you don't have any, you can [create a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md). + +### Step 1. Create a project + +1. Install the Vercel CLI: + + ```shell + npm i -g vercel@latest + ``` + +2. Create a [Next.js](https://nextjs.org/) project called `drizzle-example` using the following terminal command: + + ```shell + npx create-next-app@latest drizzle-example --ts --no-eslint --tailwind --no-src-dir --app --import-alias "@/*" + ``` + +3. Navigate to the `drizzle-example` directory: + + ```shell + cd drizzle-example + ``` + +4. Install the `drizzle-orm` and `@tidbcloud/serverless` packages: + + ```shell + npm install drizzle-orm @tidbcloud/serverless --force + ``` + +### Step 2. Set the environment + +1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Cloud Serverless cluster to go to its overview page. + +2. On the overview page, click **Connect** in the upper-right corner, select `Serverless Driver` in the **Connect With** drop-down box, and then click **Generate Password** to create a random password. + + > **Tip:** + > + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. + + The connection string looks like this: + + ``` + mysql://[username]:[password]@[host]/[database] + ``` + +### Step 3. Create an edge function + +1. Create a table in your TiDB Cloud Serverless cluster. + + You can use [SQL Editor in the TiDB Cloud console](/tidb-cloud/explore-data-with-chat2query.md) to execute SQL statements. Here is an example: + + ```sql + CREATE TABLE `test`.`users` ( + `id` BIGINT PRIMARY KEY auto_increment, + `full_name` TEXT, + `phone` VARCHAR(256) + ); + ``` + +2. In the `app` directory of your project, create a file `/api/edge-function-example/route.ts` and add the following code: + + ```ts + import { NextResponse } from 'next/server'; + import type { NextRequest } from 'next/server'; + import { connect } from '@tidbcloud/serverless'; + import { drizzle } from 'drizzle-orm/tidb-serverless'; + import { mysqlTable, serial, text, varchar } from 'drizzle-orm/mysql-core'; + export const runtime = 'edge'; + + // Initialize + const client = connect({ url: process.env.DATABASE_URL }); + const db = drizzle(client); + + // Define schema + export const users = mysqlTable('users', { + id: serial("id").primaryKey(), + fullName: text('full_name'), + phone: varchar('phone', { length: 256 }), + }); + export type User = typeof users.$inferSelect; // return type when queried + export type NewUser = typeof users.$inferInsert; // insert type + + export async function GET(request: NextRequest) { + // Insert and select data + const user: NewUser = { fullName: 'John Doe', phone: '123-456-7890' }; + await db.insert(users).values(user) + const result: User[] = await db.select().from(users); + return NextResponse.json(result); + } + ``` + +3. Test your code locally: + + ```shell + export DATABASE_URL='mysql://[username]:[password]@[host]/[database]' + next dev + ``` + +4. Navigate to `http://localhost:3000/api/edge-function-example` to get the response from your route. + +### Step 4. Deploy your code to Vercel + +1. Deploy your code to Vercel with the `DATABASE_URL` environment variable: + + ```shell + vercel -e DATABASE_URL='mysql://[username]:[password]@[host]/[database]' --prod + ``` + + After the deployment is complete, you will get the URL of your project. + +2. Navigate to the `${Your-URL}/api/edge-function-example` page to get the response from your route. + +## What's next + +- Learn more about [Drizzle](https://orm.drizzle.team/docs/overview) and [drizzle-orm/tidb-serverless](https://orm.drizzle.team/docs/get-started-mysql#tidb-serverless). +- Learn how to [integrate TiDB Cloud with Vercel](/tidb-cloud/integrate-tidbcloud-with-vercel.md). diff --git a/tidb-cloud/serverless-driver-kysely-example.md b/tidb-cloud/serverless-driver-kysely-example.md new file mode 100644 index 0000000000000..38a9763ababeb --- /dev/null +++ b/tidb-cloud/serverless-driver-kysely-example.md @@ -0,0 +1,299 @@ +--- +title: TiDB Cloud Serverless Driver Kysely Tutorial +summary: Learn how to use TiDB Cloud serverless driver with Kysely. +--- + +# TiDB Cloud Serverless Driver Kysely Tutorial + +[Kysely](https://kysely.dev/docs/intro) is a type-safe and autocompletion-friendly TypeScript SQL query builder. TiDB Cloud offers [@tidbcloud/kysely](https://github.com/tidbcloud/kysely), enabling you to use Kysely over HTTPS with [TiDB Cloud serverless driver](/tidb-cloud/serverless-driver.md). Compared with the traditional TCP way, [@tidbcloud/kysely](https://github.com/tidbcloud/kysely) brings the following benefits: + +- Better performance in serverless environments. +- Ability to use Kysely in edge environments. + +This tutorial describes how to use TiDB Cloud serverless driver with Kysely in Node.js environments and edge environments. + +## Use TiDB Cloud Kysely dialect in Node.js environments + +This section describes how to use TiDB Cloud serverless driver with Kysely in Node.js environments. + +### Before you begin + +To complete this tutorial, you need the following: + +- [Node.js](https://nodejs.org/en) >= 18.0.0. +- [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) or your preferred package manager. +- A TiDB Cloud Serverless cluster. If you don't have any, you can [create a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md). + +### Step 1. Create a project + +1. Create a project named `kysely-node-example`: + + ``` + mkdir kysely-node-example + cd kysely-node-example + ``` + +2. Install the `kysely`, `@tidbcloud/kysely`, and `@tidbcloud/serverless` packages: + + ``` + npm install kysely @tidbcloud/kysely @tidbcloud/serverless + ``` + +3. In the root directory of your project, locate the `package.json` file, and then specify the ES module by adding `"type": "module"` to the file: + + ```json + { + "type": "module", + "dependencies": { + "@tidbcloud/kysely": "^0.0.4", + "@tidbcloud/serverless": "^0.0.7", + "kysely": "^0.26.3", + } + } + ``` + +4. In the root directory of your project, add a `tsconfig.json` file to define the TypeScript compiler options. Here is an example file: + + ```json + { + "compilerOptions": { + "module": "ES2022", + "target": "ES2022", + "moduleResolution": "node", + "strict": false, + "declaration": true, + "outDir": "dist", + "removeComments": true, + "allowJs": true, + "esModuleInterop": true, + "resolveJsonModule": true + } + } + ``` + +### Step 2. Set the environment + +1. On the overview page of your TiDB Cloud Serverless cluster, click **Connect** in the upper-right corner, and then get the connection string for your database from the displayed dialog. The connection string looks like this: + + ``` + mysql://[username]:[password]@[host]/[database] + ``` + +2. Set the environment variable `DATABASE_URL` in your local environment. For example, in Linux or macOS, you can run the following command: + + ```bash + export DATABASE_URL='mysql://[username]:[password]@[host]/[database]' + ``` + +### Step 3. Use Kysely to query data + +1. Create a table in your TiDB Cloud Serverless cluster and insert some data. + + You can use [SQL Editor in the TiDB Cloud console](/tidb-cloud/explore-data-with-chat2query.md) to execute SQL statements. Here is an example: + + ```sql + CREATE TABLE `test`.`person` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) NULL DEFAULT NULL, + `gender` enum('male','female') NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE + ); + + insert into test.person values (1,'pingcap','male') + ``` + +2. In the root directory of your project, create a file named `hello-world.ts` and add the following code: + + ```ts + import { Kysely,GeneratedAlways,Selectable } from 'kysely' + import { TiDBServerlessDialect } from '@tidbcloud/kysely' + + // Types + interface Database { + person: PersonTable + } + + interface PersonTable { + id: GeneratedAlways + name: string + gender: "male" | "female" + } + + // Dialect + const db = new Kysely({ + dialect: new TiDBServerlessDialect({ + url: process.env.DATABASE_URL + }), + }) + + // Simple Querying + type Person = Selectable + export async function findPeople(criteria: Partial = {}) { + let query = db.selectFrom('person') + + if (criteria.name){ + query = query.where('name', '=', criteria.name) + } + + return await query.selectAll().execute() + } + + console.log(await findPeople()) + ``` + +### Step 4. Run the Typescript code + +1. Install `ts-node` to transform TypeScript into JavaScript, and then install `@types/node` to provide TypeScript type definitions for Node.js. + + ``` + npm install -g ts-node + npm i --save-dev @types/node + ``` + +2. Run the Typescript code with the following command: + + ``` + ts-node --esm hello-world.ts + ``` + +## Use TiDB Cloud Kysely dialect in edge environments + +This section takes the TiDB Cloud Kysely dialect in Vercel Edge Function as an example. + +### Before you begin + +To complete this tutorial, you need the following: + +- A [Vercel](https://vercel.com/docs) account that provides edge environment. +- [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) or your preferred package manager. +- A TiDB Cloud Serverless cluster. If you don't have any, you can [create a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md). + +### Step 1. Create a project + +1. Install the Vercel CLI: + + ``` + npm i -g vercel@latest + ``` + +2. Create a [Next.js](https://nextjs.org/) project called `kysely-example` using the following terminal commands: + + ``` + npx create-next-app@latest kysely-example --ts --no-eslint --tailwind --no-src-dir --app --import-alias "@/*" + cd kysely-example + ``` + +3. Install the `kysely`, `@tidbcloud/kysely`, and `@tidbcloud/serverless` packages: + + ``` + npm install kysely @tidbcloud/kysely @tidbcloud/serverless + ``` + +### Step 2. Set the environment + +On the overview page of your TiDB Cloud Serverless cluster, click **Connect** in the upper-right corner, and then get the connection string for your database from the displayed dialog. The connection string looks like this: + +``` +mysql://[username]:[password]@[host]/[database] +``` + +### Step 3. Create an edge function + +1. Create a table in your TiDB Cloud Serverless cluster and insert some data. + + You can use [SQL Editor in the TiDB Cloud console](/tidb-cloud/explore-data-with-chat2query.md) to execute SQL statements. Here is an example: + + ```sql + CREATE TABLE `test`.`person` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) NULL DEFAULT NULL, + `gender` enum('male','female') NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE + ); + + insert into test.person values (1,'pingcap','male') + ``` + +2. In the `app` directory of your project, create a file `/api/edge-function-example/route.ts` and add the following code: + + ```ts + import { NextResponse } from 'next/server'; + import type { NextRequest } from 'next/server'; + import { Kysely,GeneratedAlways,Selectable } from 'kysely' + import { TiDBServerlessDialect } from '@tidbcloud/kysely' + + export const runtime = 'edge'; + + // Types + interface Database { + person: PersonTable + } + + interface PersonTable { + id: GeneratedAlways + name: string + gender: "male" | "female" | "other" + } + + // Dialect + const db = new Kysely({ + dialect: new TiDBServerlessDialect({ + url: process.env.DATABASE_URL + }), + }) + + // Query + type Person = Selectable + async function findPeople(criteria: Partial = {}) { + let query = db.selectFrom('person') + + if (criteria.name){ + query = query.where('name', '=', criteria.name) + } + + return await query.selectAll().execute() + } + + export async function GET(request: NextRequest) { + + const searchParams = request.nextUrl.searchParams + const query = searchParams.get('query') + + let response = null; + if (query) { + response = await findPeople({name: query}) + } else { + response = await findPeople() + } + + return NextResponse.json(response); + } + ``` + + The preceding code accepts a query parameter `query` and returns the result of the query. If the query parameter is not provided, it returns all records in the `person` table. + +3. Test your code locally: + + ``` + export DATABASE_URL='mysql://[username]:[password]@[host]/[database]' + next dev + ``` + +4. Navigate to `http://localhost:3000/api/edge-function-example` to get the response from your route. + +### Step 4. Deploy your code to Vercel + +1. Deploy your code to Vercel with the `DATABASE_URL` environment variable: + + ``` + vercel -e DATABASE_URL='mysql://[username]:[password]@[host]/[database]' --prod + ``` + + After the deployment is complete, you will get the URL of your project. + +2. Navigate to the `${Your-URL}/api/edge-function-example` page to get the response from your route. + +## What's next + +- Learn more about [Kysely](https://kysely.dev/docs/intro) and [@tidbcloud/kysely](https://github.com/tidbcloud/kysely) +- Learn how to [integrate TiDB Cloud with Vercel](/tidb-cloud/integrate-tidbcloud-with-vercel.md) diff --git a/tidb-cloud/serverless-driver-node-example.md b/tidb-cloud/serverless-driver-node-example.md new file mode 100644 index 0000000000000..e900f653bd4e9 --- /dev/null +++ b/tidb-cloud/serverless-driver-node-example.md @@ -0,0 +1,95 @@ +--- +title: TiDB Cloud Serverless Driver Node.js Tutorial +summary: Learn how to use TiDB Cloud serverless driver in a local Node.js project. +--- + +# TiDB Cloud Serverless Driver Node.js Tutorial + +This tutorial describes how to use TiDB Cloud serverless driver in a local Node.js project. + +> **Note:** +> +> - This tutorial is applicable to TiDB Cloud Serverless clusters only. +> - To learn how to use TiDB Cloud serverless driver with Cloudflare Workers, Vercel Edge Functions, and Netlify Edge Functions, check out our [Insights into Automotive Sales](https://car-sales-insight.vercel.app/) and the [sample repository](https://github.com/tidbcloud/car-sales-insight). + +## Before you begin + +To complete this step-by-step tutorial, you need the following: + +- [Node.js](https://nodejs.org/en) >= 18.0.0. +- [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) or your preferred package manager. +- A TiDB Cloud Serverless cluster. If you don't have any, you can [create a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md). + +## Step 1. Create a local Node.js project + +1. Create a project named `node-example`: + + ```shell + mkdir node-example + cd node-example + ``` + +2. Install the TiDB Cloud serverless driver using npm or your preferred package manager. + + The following command takes installation with npm as an example. Executing this command will create a `node_modules` directory and a `package.json` file in your project directory. + + ``` + npm install @tidbcloud/serverless + ``` + +## Step 2. Use the serverless driver + +The serverless driver supports both CommonJS and ES modules. The following steps take the usage of the ES module as an example. + +1. On the overview page of your TiDB Cloud Serverless cluster, click **Connect** in the upper-right corner, and then get the connection string for your database from the displayed dialog. The connection string looks like this: + + ``` + mysql://[username]:[password]@[host]/[database] + ``` + +2. In the `package.json` file, specify the ES module by adding `type: "module"`. + + For example: + + ```json + { + "type": "module", + "dependencies": { + "@tidbcloud/serverless": "^0.0.7", + } + } + ``` + +3. Create a file named `index.js` in your project directory and add the following code: + + ```js + import { connect } from '@tidbcloud/serverless' + + const conn = connect({url: 'mysql://[username]:[password]@[host]/[database]'}) // replace with your TiDB Cloud Serverless cluster information + console.log(await conn.execute("show tables")) + ``` + +4. Run your project with the following command: + + ``` + node index.js + ``` + +## Compatibility with earlier versions of Node.js + +If you are using Node.js earlier than 18.0.0, which does not have a global `fetch` function, you can take the following steps to get `fetch`: + +1. Install a package that provides `fetch`, such as `undici`: + + ``` + npm install undici + ``` + +2. Pass the `fetch` function to the `connect` function: + + ```js + import { connect } from '@tidbcloud/serverless' + import { fetch } from 'undici' + + const conn = connect({url: 'mysql://[username]:[password]@[host]/[database]',fetch}) + ``` \ No newline at end of file diff --git a/tidb-cloud/serverless-driver-prisma-example.md b/tidb-cloud/serverless-driver-prisma-example.md new file mode 100644 index 0000000000000..abaa550b70e4d --- /dev/null +++ b/tidb-cloud/serverless-driver-prisma-example.md @@ -0,0 +1,268 @@ +--- +title: TiDB Cloud Serverless Driver Prisma Tutorial +summary: Learn how to use TiDB Cloud serverless driver with Prisma ORM. +--- + +# TiDB Cloud Serverless Driver Prisma Tutorial + +[Prisma](https://www.prisma.io/docs) is an open source next-generation ORM (Object-Relational Mapping) that helps developers interact with their database in an intuitive, efficient, and safe way. TiDB Cloud offers [@tidbcloud/prisma-adapter](https://github.com/tidbcloud/prisma-adapter), enabling you to use [Prisma Client](https://www.prisma.io/docs/concepts/components/prisma-client) over HTTPS with [TiDB Cloud serverless driver](/tidb-cloud/serverless-driver.md). Compared with the traditional TCP way, [@tidbcloud/prisma-adapter](https://github.com/tidbcloud/prisma-adapter) brings the following benefits: + +- Better performance of Prisma Client in serverless environments +- Ability to use Prisma Client in edge environments + +This tutorial describes how to use [@tidbcloud/prisma-adapter](https://github.com/tidbcloud/prisma-adapter) in serverless environments and edge environments. + +## Install + +You need to install both [@tidbcloud/prisma-adapter](https://github.com/tidbcloud/prisma-adapter) and [TiDB Cloud serverless driver](/tidb-cloud/serverless-driver.md). You can install them using [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) or your preferred package manager. + +Taking npm as an example, you can run the following commands for installation: + +```shell +npm install @tidbcloud/prisma-adapter +npm install @tidbcloud/serverless +``` + +## Enable `driverAdapters` + +To use the Prisma adapter, you need to enable the `driverAdapters` feature in the `schema.prisma` file. For example: + +```prisma +generator client { + provider = "prisma-client-js" + previewFeatures = ["driverAdapters"] +} + +datasource db { + provider = "mysql" + url = env("DATABASE_URL") +} +``` + +## Initialize Prisma Client + +Before using Prisma Client, you need to initialize it with `@tidbcloud/prisma-adapter`. For example: + +```js +import { connect } from '@tidbcloud/serverless'; +import { PrismaTiDBCloud } from '@tidbcloud/prisma-adapter'; +import { PrismaClient } from '@prisma/client'; + +// Initialize Prisma Client +const connection = connect({ url: ${DATABASE_URL} }); +const adapter = new PrismaTiDBCloud(connection); +const prisma = new PrismaClient({ adapter }); +``` + +Then, queries from Prisma Client can be sent to the TiDB Cloud serverless driver for processing. + +## Use the Prisma adapter in Node.js environments + +This section provides an example of how to use `@tidbcloud/prisma-adapter` in Node.js environments. + +### Before you begin + +To complete this tutorial, you need the following: + +- [Node.js](https://nodejs.org/en) >= 18.0.0. +- [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) or your preferred package manager. +- A TiDB Cloud Serverless cluster. If you don't have any, you can [create a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md). + +### Step 1. Create a project + +1. Create a project named `prisma-example`: + + ``` + mkdir prisma-example + cd prisma-example + ``` + +2. Install the `@tidbcloud/prisma-adapter` driver adapter, the `@tidbcloud/serverless` serverless driver, and the Prisma CLI. + + The following commands use npm as the package manager. Executing `npm install @tidbcloud/serverless` will create a `node_modules` directory and a `package.json` file in your project directory. + + ``` + npm install @tidbcloud/prisma-adapter + npm install @tidbcloud/serverless + npm install prisma --save-dev + ``` + +3. In the `package.json` file, specify the ES module by adding `type: "module"`: + + ```json + { + "type": "module", + "dependencies": { + "@prisma/client": "^5.5.2", + "@tidbcloud/prisma-adapter": "^5.5.2", + "@tidbcloud/serverless": "^0.0.7" + }, + "devDependencies": { + "prisma": "^5.5.2" + } + } + ``` + +### Step 2. Set the environment + +1. On the overview page of your TiDB Cloud Serverless cluster, click **Connect** in the upper-right corner, and then get the connection string for your database from the displayed dialog. The connection string looks like this: + + ``` + mysql://[username]:[password]@[host]:4000/[database]?sslaccept=strict + ``` + +2. In the root directory of your project, create a file named `.env`, define an environment variable named `DATABASE_URL` as follows, and then replace the placeholders `[]` in this variable with the corresponding parameters in the connection string. + + ```dotenv + DATABASE_URL='mysql://[username]:[password]@[host]:4000/[database]?sslaccept=strict' + ``` + + > **Note:** + > + > `@tidbcloud/prisma-adapter` only supports the use of Prisma Client over HTTPS. For [Prisma Migrate](https://www.prisma.io/docs/concepts/components/prisma-migrate) and [Prisma Introspection](https://www.prisma.io/docs/concepts/components/introspection), the traditional TCP connection is still used. If you only need to use Prisma Client, you can simplify `DATABASE_URL` to the `mysql://[username]:[password]@[host]/[database]` format. + +3. Install `dotenv` to load the environment variable from the `.env` file: + + ``` + npm install dotenv + ``` + +### Step 3. Define your schema + +1. Create a file named `schema.prisma`. In this file, include the `driverAdapters` preview feature and reference the `DATABASE_URL` environment variable. Here is an example file: + + ``` + // schema.prisma + generator client { + provider = "prisma-client-js" + previewFeatures = ["driverAdapters"] + } + + datasource db { + provider = "mysql" + url = env("DATABASE_URL") + } + ``` + +2. In the `schema.prisma` file, define a data model for your database table. In the following example, a data model named `user` is defined. + + ``` + // schema.prisma + generator client { + provider = "prisma-client-js" + previewFeatures = ["driverAdapters"] + } + + datasource db { + provider = "mysql" + url = env("DATABASE_URL") + } + + // define a data model according to your database table + model user { + id Int @id @default(autoincrement()) + email String? @unique(map: "uniq_email") @db.VarChar(255) + name String? @db.VarChar(255) + } + ``` + +3. Synchronize your database with the Prisma schema. You can either manually create the database tables in your TiDB Cloud Serverless cluster or use the Prisma CLI to create them automatically as follows: + + ``` + npx prisma db push + ``` + + This command will create the `user` table in your TiDB Cloud Serverless cluster through the traditional TCP connection, rather than through the HTTPS connection using `@tidbcloud/prisma-adapter`. This is because it uses the same engine as Prisma Migrate. For more information about this command, see [Prototype your schema](https://www.prisma.io/docs/concepts/components/prisma-migrate/db-push). + +4. Generate Prisma Client: + + ``` + npx prisma generate + ``` + + This command will generate Prisma Client based on the Prisma schema. + +### Step 4. Execute CRUD operations + +1. Create a file named `hello-word.js` and add the following code to initialize Prisma Client: + + ```js + import { connect } from '@tidbcloud/serverless'; + import { PrismaTiDBCloud } from '@tidbcloud/prisma-adapter'; + import { PrismaClient } from '@prisma/client'; + import dotenv from 'dotenv'; + + // setup + dotenv.config(); + const connectionString = `${process.env.DATABASE_URL}`; + + // Initialize Prisma Client + const connection = connect({ url: connectionString }); + const adapter = new PrismaTiDBCloud(connection); + const prisma = new PrismaClient({ adapter }); + ``` + +2. Execute some CRUD operations with Prisma Client. For example: + + ```js + // Insert + const user = await prisma.user.create({ + data: { + email: 'test@pingcap.com', + name: 'test', + }, + }) + console.log(user) + + // Query + console.log(await prisma.user.findMany()) + + // Delete + await prisma.user.delete({ + where: { + id: user.id, + }, + }) + ``` + +3. Execute some transaction operations with Prisma Client. For example: + + ```js + const createUser1 = prisma.user.create({ + data: { + email: 'test1@pingcap.com', + name: 'test1', + }, + }) + const createUser2 = prisma.user.create({ + data: { + email: 'test1@pingcap.com', + name: 'test1', + }, + }) + const createUser3 = prisma.user.create({ + data: { + email: 'test2@pingcap.com', + name: 'test2', + }, + }) + + try { + await prisma.$transaction([createUser1, createUser2]) // Operations fail because the email address is duplicated + } catch (e) { + console.log(e) + } + + try { + await prisma.$transaction([createUser2, createUser3]) // Operations success because the email address is unique + } catch (e) { + console.log(e) + } + ``` + +## Use the Prisma adapter in edge environments + +You can use `@tidbcloud/prisma-adapter` v5.11.0 or a later version in edge environments such as Vercel Edge Functions and Cloudflare Workers. + +- [Vercel Edge Function example](https://github.com/tidbcloud/serverless-driver-example/tree/main/prisma/prisma-vercel-example) +- [Cloudflare Workers example](https://github.com/tidbcloud/serverless-driver-example/tree/main/prisma/prisma-cloudflare-worker-example) diff --git a/tidb-cloud/serverless-driver.md b/tidb-cloud/serverless-driver.md new file mode 100644 index 0000000000000..c3ebd151db135 --- /dev/null +++ b/tidb-cloud/serverless-driver.md @@ -0,0 +1,342 @@ +--- +title: TiDB Cloud Serverless Driver (Beta) +summary: Learn how to connect to TiDB Cloud Serverless from serverless and edge environments. +aliases: ['/tidbcloud/serverless-driver-config'] +--- + +# TiDB Cloud Serverless Driver (Beta) + +## Why use TiDB Cloud Serverless Driver (Beta) + +Traditional TCP-based MySQL drivers are not suitable for serverless functions due to their expectation of long-lived, persistent TCP connections, which contradict the short-lived nature of serverless functions. Moreover, in edge environments such as [Vercel Edge Functions](https://vercel.com/docs/functions/edge-functions) and [Cloudflare Workers](https://workers.cloudflare.com/), where comprehensive TCP support and full Node.js compatibility may be lacking, these drivers may not work at all. + +[TiDB Cloud serverless driver (Beta)](https://github.com/tidbcloud/serverless-js) for JavaScript allows you to connect to your TiDB Cloud Serverless cluster over HTTP, which is generally supported by serverless environments. With it, it is now possible to connect to TiDB Cloud Serverless clusters from edge environments and reduce connection overhead with TCP while keeping the similar development experience of traditional TCP-based MySQL drivers. + +> **Note:** +> +> If you prefer programming with RESTful API rather than SQL or ORM, you can use [Data Service (beta)](/tidb-cloud/data-service-overview.md). + +## Install the serverless driver + +You can install the driver with npm: + +```bash +npm install @tidbcloud/serverless +``` + +## Use the serverless driver + +You can use the serverless driver to query data of a TiDB Cloud Serverless cluster or perform interactive transactions. + +### Query + +To query data from a TiDB Cloud Serverless cluster, you need to create a connection first. Then you can use the connection to execute raw SQL queries. For example: + +```ts +import { connect } from '@tidbcloud/serverless' + +const conn = connect({url: 'mysql://[username]:[password]@[host]/[database]'}) +const results = await conn.execute('select * from test where id = ?',[1]) +``` + +### Transaction (experimental) + +You can also perform interactive transactions with the serverless driver. For example: + +```ts +import { connect } from '@tidbcloud/serverless' + +const conn = connect({url: 'mysql://[username]:[password]@[host]/[database]'}) +const tx = await conn.begin() + +try { + await tx.execute('insert into test values (1)') + await tx.execute('select * from test') + await tx.commit() +} catch (err) { + await tx.rollback() + throw err +} +``` + +## Edge examples + +Here are some examples of using the serverless driver in edge environments. For a complete example, you can also try this [live demo](https://github.com/tidbcloud/car-sales-insight). + + + +
    + +```ts +import { NextResponse } from 'next/server'; +import type { NextRequest } from 'next/server'; +import { connect } from '@tidbcloud/serverless' +export const runtime = 'edge' + +export async function GET(request: NextRequest) { + const conn = connect({url: process.env.DATABASE_URL}) + const result = await conn.execute('show tables') + return NextResponse.json({result}); +} +``` + +Learn more about [using TiDB Cloud serverless driver in Vercel](/tidb-cloud/integrate-tidbcloud-with-vercel.md). + +
    + +
    + +```ts +import { connect } from '@tidbcloud/serverless' +export interface Env { + DATABASE_URL: string; +} +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { + const conn = connect({url: env.DATABASE_URL}) + const result = await conn.execute('show tables') + return new Response(JSON.stringify(result)); + }, +}; +``` + +Learn more about [using TiDB Cloud serverless driver in Cloudflare Workers](/tidb-cloud/integrate-tidbcloud-with-cloudflare.md). + +
    + +
    + +```ts +import { connect } from 'https://esm.sh/@tidbcloud/serverless' + +export default async () => { + const conn = connect({url: Netlify.env.get('DATABASE_URL')}) + const result = await conn.execute('show tables') + return new Response(JSON.stringify(result)); +} +``` + +Learn more about [using TiDB Cloud serverless driver in Netlify](/tidb-cloud/integrate-tidbcloud-with-netlify.md#use-the-edge-function). + +
    + +
    + +```ts +import { connect } from "npm:@tidbcloud/serverless" + +const conn = connect({url: Deno.env.get('DATABASE_URL')}) +const result = await conn.execute('show tables') +``` + +
    + +
    + +```ts +import { connect } from "@tidbcloud/serverless" + +const conn = connect({url: Bun.env.DATABASE_URL}) +const result = await conn.execute('show tables') +``` + +
    + +
    + +## Configure the serverless driver + +You can configure TiDB Cloud serverless driver at both the connection level and the SQL level. + +### Connection level configurations + +At the connection level, you can make the following configurations: + +| Name | Type | Default value | Description | +|--------------|----------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `username` | string | N/A | Username of TiDB Cloud Serverless | +| `password` | string | N/A | Password of TiDB Cloud Serverless | +| `host` | string | N/A | Hostname of TiDB Cloud Serverless | +| `database` | string | `test` | Database of TiDB Cloud Serverless | +| `url` | string | N/A | The URL for the database, in the `mysql://[username]:[password]@[host]/[database]` format, where `database` can be skipped if you intend to connect to the default database. | +| `fetch` | function | global fetch | Custom fetch function. For example, you can use the `undici` fetch in node.js. | +| `arrayMode` | bool | `false` | Whether to return results as arrays instead of objects. To get better performance, set it to `true`. | +| `fullResult` | bool | `false` | Whether to return full result object instead of just rows. To get more detailed results, set it to `true`. | +| `decoders` | object | `{}` | A collection of key-value pairs, which enables you to customize the decoding process for different column types. In each pair, you can specify a column type as the key and specify a corresponding function as the value. This function takes the raw string value received from TiDB Cloud serverless driver as an argument and returns the decoded value. | + +**Database URL** + +> **Note:** +> +> If your username, password, or database name contains special characters, you must [percentage-encode](https://en.wikipedia.org/wiki/Percent-encoding) these characters when passing them by the URL. For example, the password `password1@//?` needs to be encoded as `password1%40%2F%2F%3F` in the URL. + +When `url` is configured, there is no need to configure `host`, `username`, `password`, and `database` separately. The following codes are equivalent: + +```ts +const config = { + host: '', + username: '', + password: '', + database: '', + arrayMode: true, +} + +const conn = connect(config) +``` + +```ts +const config = { + url: process.env['DATABASE_URL'] || 'mysql://[username]:[password]@[host]/[database]', + arrayMode: true +} + +const conn = connect(config) +``` + +### SQL level options + +> **Note:** +> +> The SQL level options have a higher priority over connection level configurations. + +At the SQL level, you can configure the following options: + +| Option | Type | Default value | Description | +|--------------|--------|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `arrayMode` | bool | `false` | Whether to return results as arrays instead of objects. To get better performance, set it to `true`. | +| `fullResult` | bool | `false` | Whether to return the full result object instead of just rows. To get more detailed results, set it to `true`. | +| `isolation` | string | `REPEATABLE READ` | The transaction isolation level, which can be set to `READ COMMITTED` or `REPEATABLE READ`. | +| `decoders` | object | `{}` | A collection of key-value pairs, which enables you to customize the decoding process for different column types. In each pair, you can specify a column type as the key and specify a corresponding function as the value. This function takes the raw string value received from TiDB Cloud serverless driver as an argument and returns the decoded value. If you have configured `decoders` at both the connection and SQL levels, the key-value pairs with different keys configured at the connection level will be merged to the SQL level to take effect. If the same key (this is, column type) is specified at both levels, the value at the SQL level takes precedence. | + +**arrayMode and fullResult** + +To return the full result object as arrays, you can configure the `arrayMode` and `fullResult` options as follows: + +```ts +const conn = connect({url: process.env['DATABASE_URL'] || 'mysql://[username]:[password]@[host]/[database]'}) +const results = await conn.execute('select * from test',null,{arrayMode:true,fullResult:true}) +``` + +**isolation** + +The `isolation` option can only be used in the `begin` function. + +```ts +const conn = connect({url: 'mysql://[username]:[password]@[host]/[database]'}) +const tx = await conn.begin({isolation:"READ COMMITTED"}) +``` + +**decoders** + +To customize the format of returned column values, you can configure the `decoder` option in the `connect()` method as follows: + +```ts +import { connect, ColumnType } from '@tidbcloud/serverless'; + +const conn = connect({ + url: 'mysql://[username]:[password]@[host]/[database]', + decoders: { + // By default, TiDB Cloud serverless driver returns the BIGINT type as text value. This decoder converts BIGINT to the JavaScript built-in BigInt type. + [ColumnType.BIGINT]: (rawValue: string) => BigInt(rawValue), + + // By default, TiDB Cloud serverless driver returns the DATETIME type as the text value in the 'yyyy-MM-dd HH:mm:ss' format. This decoder converts the DATETIME text to the JavaScript native Date object. + [ColumnType.DATETIME]: (rawValue: string) => new Date(rawValue), + } +}) + +// You can also configure the decoder option at the SQL level to override the decoders with the same keys at the connection level. +conn.execute(`select ...`, [], { + decoders: { + // ... + } +}) +``` + +> **Note:** +> +> TiDB Cloud serverless driver configuration changes: +> +> - v0.0.7: add the SQL level option `isolation`. +> - v0.0.10: add the connection level configuration `decoders` and the SQL level option `decoders`. + +## Features + +### Supported SQL statements + +DDL is supported and the following SQL statements are supported: `SELECT`, `SHOW`, `EXPLAIN`, `USE`, `INSERT`, `UPDATE`, `DELETE`, `BEGIN`, `COMMIT`, `ROLLBACK`, and `SET`. + +### Data type mapping + +The type mapping between TiDB Cloud Serverless and Javascript is as follows: + +| TiDB Cloud Serverless type | Javascript type | +|----------------------|-----------------| +| TINYINT | number | +| UNSIGNED TINYINT | number | +| BOOL | number | +| SMALLINT | number | +| UNSIGNED SMALLINT | number | +| MEDIUMINT | number | +| INT | number | +| UNSIGNED INT | number | +| YEAR | number | +| FLOAT | number | +| DOUBLE | number | +| BIGINT | string | +| UNSIGNED BIGINT | string | +| DECIMAL | string | +| CHAR | string | +| VARCHAR | string | +| BINARY | Uint8Array | +| VARBINARY | Uint8Array | +| TINYTEXT | string | +| TEXT | string | +| MEDIUMTEXT | string | +| LONGTEXT | string | +| TINYBLOB | Uint8Array | +| BLOB | Uint8Array | +| MEDIUMBLOB | Uint8Array | +| LONGBLOB | Uint8Array | +| DATE | string | +| TIME | string | +| DATETIME | string | +| TIMESTAMP | string | +| ENUM | string | +| SET | string | +| BIT | Uint8Array | +| JSON | object | +| NULL | null | +| Others | string | + +> **Note:** +> +> Make sure to use the default `utf8mb4` character set in TiDB Cloud Serverless for the type conversion to JavaScript strings, because TiDB Cloud serverless driver uses the UTF-8 encoding to decode them to strings. + +> **Note:** +> +> TiDB Cloud serverless driver data type mapping changes: +> +> - v0.1.0: the `BINARY`, `VARBINARY`, `TINYBLOB`, `BLOB`, `MEDIUMBLOB`, `LONGBLOB`, and `BIT` types are now returned as a `Uint8Array` instead of a `string`. + +### ORM integrations + +TiDB Cloud serverless driver has been integrated with the following ORMs: + +- [TiDB Cloud serverless driver Kysely dialect](https://github.com/tidbcloud/kysely). +- [TiDB Cloud serverless driver Prisma adapter](https://github.com/tidbcloud/prisma-adapter). + +## Pricing + +The serverless driver itself is free, but accessing data with the driver generates [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit) and storage usage. The pricing follows the [TiDB Cloud Serverless pricing](https://www.pingcap.com/tidb-serverless-pricing-details/) model. + +## Limitations + +Currently, using serverless driver has the following limitations: + +- Up to 10,000 rows can be fetched in a single query. +- You can execute only a single SQL statement at a time. Multiple SQL statements in one query are not supported yet. +- Connection with [private endpoints](/tidb-cloud/set-up-private-endpoint-connections-serverless.md) is not supported yet. + +## What's next + +- Learn how to [use TiDB Cloud serverless driver in a local Node.js project](/tidb-cloud/serverless-driver-node-example.md). diff --git a/tidb-cloud/serverless-export.md b/tidb-cloud/serverless-export.md new file mode 100644 index 0000000000000..16f728f490b3c --- /dev/null +++ b/tidb-cloud/serverless-export.md @@ -0,0 +1,315 @@ +--- +title: Export Data from TiDB Cloud Serverless +summary: Learn how to export data from TiDB Cloud Serverless clusters. +--- + +# Export Data from TiDB Cloud Serverless + +TiDB Cloud Serverless Export (Beta) is a service that enables you to export data from a TiDB Cloud Serverless cluster to a local file or an external storage service. You can use the exported data for backup, migration, data analysis, or other purposes. + +While you can also export data using tools such as [mysqldump](https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html) and TiDB [Dumpling](https://docs.pingcap.com/tidb/dev/dumpling-overview), TiDB Cloud Serverless Export offers a more convenient and efficient way to export data from a TiDB Cloud Serverless cluster. It brings the following benefits: + +- Convenience: the export service provides a simple and easy-to-use way to export data from a TiDB Cloud Serverless cluster, eliminating the need for additional tools or resources. +- Isolation: the export service uses separate computing resources, ensuring isolation from the resources used by your online services. +- Consistency: the export service ensures the consistency of the exported data without causing locks, which does not affect your online services. + +## Export locations + +You can export data to the following locations: + +- A local file +- An external storage, including: + + - [Amazon S3](https://aws.amazon.com/s3/) + - [Google Cloud Storage](https://cloud.google.com/storage) + - [Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/) + +> **Note:** +> +> If the size of the data to be exported is large (more than 100 GiB), it is recommended that you export it to an external storage. + +### A local file + +To export data from a TiDB Cloud Serverless cluster to a local file, you need to export data [using the TiDB Cloud console](#export-data-to-a-local-file) or [using the TiDB Cloud CLI](/tidb-cloud/ticloud-serverless-export-create.md), and then download the exported data using the TiDB Cloud CLI. + +Exporting data to a local file has the following limitations: + +- Downloading exported data using the TiDB Cloud console is not supported. +- Exported data is saved in the stashing area of TiDB Cloud and will expire after two days. You need to download the exported data in time. +- If the storage space of stashing area is full, you will not be able to export data to the local file. + +### Amazon S3 + +To export data to Amazon S3, you need to provide the following information: + +- URI: `s3:///` +- One of the following access credentials: + - [An access key](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html): make sure the access key has the `s3:PutObject` and `s3:ListBucket` permissions. + - [A role ARN](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference-arns.html): make sure the role ARN has the `s3:PutObject` and `s3:ListBucket` permissions. + +For more information, see [Configure External Storage Access for TiDB Cloud Serverless](/tidb-cloud/serverless-external-storage.md#configure-amazon-s3-access). + +### Google Cloud Storage + +To export data to Google Cloud Storage, you need to provide the following information: + +- URI: `gs:///` +- Access credential: a **base64 encoded** [service account key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) for your bucket. Make sure the service account key has the `storage.objects.create` permission. + +For more information, see [Configure External Storage Access for TiDB Serverless](/tidb-cloud/serverless-external-storage.md#configure-gcs-access). + +> **Note:** +> +> Currently, you can only export to Google Cloud Storage using [TiDB Cloud CLI](/tidb-cloud/cli-reference.md). + +### Azure Blob Storage + +To export data to Azure Blob Storage, you need to provide the following information: + +- URI: `azure://.blob.core.windows.net//` +- Access credential: a [shared access signature (SAS) token](https://docs.microsoft.com/en-us/azure/storage/common/storage-sas-overview) for your Azure Blob Storage container. Make sure the SAS token has the `Read` and `Write` permissions on the `Container` and `Object` resources. + +For more information, see [Configure External Storage Access for TiDB Serverless](/tidb-cloud/serverless-external-storage.md#configure-azure-blob-storage-access). + +> **Note:** +> +> Currently, you can only export to Azure Blob Storage using [TiDB Cloud CLI](/tidb-cloud/cli-reference.md). + +## Export options + +### Data filtering + +- TiDB Cloud console supports exporting data with the selected databases and tables. +- TiDB Cloud CLI supports exporting data with SQL statements and [table filters](/table-filter.md). + +### Data formats + +You can export data in the following formats: + +- `SQL`: export data in SQL format. +- `CSV`: export data in CSV format. You can specify the following options: + - `delimiter`: specify the delimiter used in the exported data. The default delimiter is `"`. + - `separator`: specify the character used to separate fields in the exported data. The default separator is `,`. + - `header`: specify whether to include a header row in the exported data. The default value is `true`. + - `null-value`: specify the string that represents a NULL value in the exported data. The default value is `\N`. +- `Parquet`: export data in Parquet format. Currently, it is only supported in TiDB Cloud CLI. + +The schema and data are exported according to the following naming conventions: + +| Item | Not compressed | Compressed | +|-----------------|-------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------| +| Database schema | {database}-schema-create.sql | {database}-schema-create.sql.{compression-type} | +| Table schema | {database}.{table}-schema.sql | {database}.{table}-schema.sql.{compression-type} | +| Data | {database}.{table}.{0001}.{csv|parquet|sql} | {database}.{table}.{0001}.{csv|sql}.{compression-type}
    {database}.{table}.{0001}.{compression-type}.parquet | + +### Data compression + +You can compress the exported CSV and SQL data using the following algorithms: + +- `gzip` (default): compress the exported data with `gzip`. +- `snappy`: compress the exported data with `snappy`. +- `zstd`: compress the exported data with `zstd`. +- `none`: do not compress the exported `data`. + +You can compress the exported Parquet data using the following algorithms: + +- `zstd` (default): compress the Parquet file with `zstd`. +- `gzip`: compress the Parquet file with `gzip`. +- `snappy`: compress the Parquet file with `snappy`. +- `none`: do not compress the Parquet file. + +### Data conversion + +When exporting data to the Parquet format, the data conversion between TiDB Cloud Serverless and Parquet is as follows: + +| TiDB Cloud Serverless Type | Parquest primitive type | Parquet logical type | +|----------------------------|-------------------------|----------------------------------------------| +| VARCHAR | BYTE_ARRAY | String(UTF8) | +| TIME | BYTE_ARRAY | String(UTF8) | +| TINYTEXT | BYTE_ARRAY | String(UTF8) | +| MEDIUMTEXT | BYTE_ARRAY | String(UTF8) | +| TEXT | BYTE_ARRAY | String(UTF8) | +| LONGTEXT | BYTE_ARRAY | String(UTF8) | +| SET | BYTE_ARRAY | String(UTF8) | +| JSON | BYTE_ARRAY | String(UTF8) | +| DATE | BYTE_ARRAY | String(UTF8) | +| CHAR | BYTE_ARRAY | String(UTF8) | +| VECTOR | BYTE_ARRAY | String(UTF8) | +| DECIMAL(1<=p<=9) | INT32 | DECIMAL(p,s) | +| DECIMAL(10<=p<=18) | INT64 | DECIMAL(p,s) | +| DECIMAL(p>=19) | BYTE_ARRAY | String(UTF8) | +| ENUM | BYTE_ARRAY | String(UTF8) | +| TIMESTAMP | INT64 | TIMESTAMP(unit=MICROS,isAdjustedToUTC=false) | +| DATETIME | INT64 | TIMESTAMP(unit=MICROS,isAdjustedToUTC=false) | +| YEAR | INT32 | / | +| TINYINT | INT32 | / | +| UNSIGNED TINYINT | INT32 | / | +| SMALLINT | INT32 | / | +| UNSIGNED SMALLINT | INT32 | / | +| MEDIUMINT | INT32 | / | +| UNSIGNED MEDIUMINT | INT32 | / | +| INT | INT32 | / | +| UNSIGNED INT | FIXED_LEN_BYTE_ARRAY(9) | DECIMAL(20,0) | +| BIGINT | FIXED_LEN_BYTE_ARRAY(9) | DECIMAL(20,0) | +| UNSIGNED BIGINT | BYTE_ARRAY | String(UTF8) | +| FLOAT | FLOAT | / | +| DOUBLE | DOUBLE | / | +| BLOB | BYTE_ARRAY | / | +| TINYBLOB | BYTE_ARRAY | / | +| MEDIUMBLOB | BYTE_ARRAY | / | +| LONGBLOB | BYTE_ARRAY | / | +| BINARY | BYTE_ARRAY | / | +| VARBINARY | BYTE_ARRAY | / | +| BIT | BYTE_ARRAY | / | + +## Examples + +### Export data to a local file + + +
    + +1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + +2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +3. On the **Import** page, click **Export Data to** in the upper-right corner, then choose **Local File** from the drop-down list. Fill in the following parameters: + + - **Task Name**: enter a name for the export task. The default value is `SNAPSHOT_{snapshot_time}`. + - **Exported Data**: choose the databases and tables you want to export. + - **Data Format**: choose **SQL File** or **CSV**. + - **Compression**: choose **Gzip**, **Snappy**, **Zstd**, or **None**. + + > **Tip:** + > + > If your cluster has neither imported nor exported any data before, you need to click **Click here to export data to...** at the bottom of the page to export data. + +4. Click **Export**. + +5. After the export task is successful, you can copy the download command displayed in the export task detail, and then download the exported data by running the command in the [TiDB Cloud CLI](/tidb-cloud/cli-reference.md). + +
    + +
    + +1. Create an export task: + + ```shell + ticloud serverless export create -c + ``` + + You will get an export ID from the output. + +2. After the export task is successful, download the exported data to your local file: + + ```shell + ticloud serverless export download -c -e + ``` + + For more information about the download command, see [ticloud serverless export download](/tidb-cloud/ticloud-serverless-export-download.md). + +
    +
    + +### Export data to Amazon S3 + + +
    + +1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + +2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +3. On the **Import** page, click **Export Data to** in the upper-right corner, then choose **Amazon S3** from the drop-down list. Fill in the following parameters: + + - **Task Name**: enter a name for the export task. The default value is `SNAPSHOT_{snapshot_time}`. + - **Exported Data**: choose the databases and tables you want to export. + - **Data Format**: choose **SQL File** or **CSV**. + - **Compression**: choose **Gzip**, **Snappy**, **Zstd**, or **None**. + - **Folder URI**: enter the URI of the Amazon S3 with the `s3:////` format. + - **Bucket Access**: choose one of the following access credentials and then fill in the credential information. If you do not have such information, see [Configure External Storage Access for TiDB Cloud Serverless](/tidb-cloud/serverless-external-storage.md#configure-amazon-s3-access). + - **AWS Role ARN**: enter the role ARN that has the `s3:PutObject` and `s3:ListBucket` permissions to access the bucket. + - **AWS Access Key**: enter the access key ID and access key secret that have the `s3:PutObject` and `s3:ListBucket` permissions to access the bucket. + +4. Click **Export**. + +
    + +
    + +```shell +ticloud serverless export create -c --s3.uri --s3.access-key-id --s3.secret-access-key --filter "database.table" +``` + +- `s3.uri`: the Amazon S3 URI with the `s3:///` format. +- `s3.access-key-id`: the access key ID of the user who has the permission to access the bucket. +- `s3.secret-access-key`: the access key secret of the user who has the permission to access the bucket. + +
    +
    + +### Export data to Google Cloud Storage + +Currently, you can only export data to Google Cloud Storage using [TiDB Cloud CLI](/tidb-cloud/cli-reference.md). + +```shell +ticloud serverless export create -c --gcs.uri --gcs.service-account-key --filter "database.table" +``` + +- `gcs.uri`: the URI of the Google Cloud Storage bucket in the `gs:///` format. +- `gcs.service-account-key`: the base64 encoded service account key. + +### Export data to Azure Blob Storage + +Currently, you can only export data to Azure Blob Storage using [TiDB Cloud CLI](/tidb-cloud/cli-reference.md). + +```shell +ticloud serverless export create -c --azblob.uri --azblob.sas-token --filter "database.table" +``` + +- `azblob.uri`: the URI of the Azure Blob Storage in the `azure://.blob.core.windows.net//` format. +- `azblob.sas-token`: the account SAS token of the Azure Blob Storage. + +### Cancel an export task + +To cancel an ongoing export task, take the following steps: + + +
    + +1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + +2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +3. On the **Import** page, click **Export** to view the export task list. + +4. Choose the export task you want to cancel, and then click **Action**. + +5. Choose **Cancel** in the drop-down list. Note that you can only cancel the export task that is in the **Running** status. + +
    + +
    + +```shell +ticloud serverless export cancel -c -e +``` + +
    +
    + +## Pricing + +The export service is free during the beta period. You only need to pay for the [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit) generated during the export process of successful or canceled tasks. For failed export tasks, you will not be charged. diff --git a/tidb-cloud/serverless-external-storage.md b/tidb-cloud/serverless-external-storage.md new file mode 100644 index 0000000000000..aafdf53005dbe --- /dev/null +++ b/tidb-cloud/serverless-external-storage.md @@ -0,0 +1,224 @@ +--- +title: Configure TiDB Cloud Serverless External Storage Access +summary: Learn how to configure Amazon Simple Storage Service (Amazon S3) access. +--- + +# Configure External Storage Access for TiDB Cloud Serverless + +If you want to import data from or export data to an external storage in a TiDB Cloud Serverless cluster, you need to configure cross-account access. This document describes how to configure access to an external storage for TiDB Cloud Serverless clusters. + +If you need to configure these external storages for a TiDB Cloud Dedicated cluster, see [Configure External Storage Access for TiDB Cloud Dedicated](/tidb-cloud/config-s3-and-gcs-access.md). + +## Configure Amazon S3 access + +To allow a TiDB Cloud Serverless cluster to access your Amazon S3 bucket, you need to configure the bucket access for the cluster. You can use either of the following methods to configure the bucket access: + +- Use a Role ARN: use a Role ARN to access your Amazon S3 bucket. +- Use an AWS access key: use the access key of an IAM user to access your Amazon S3 bucket. + + +
    + +It is recommended that you use [AWS CloudFormation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/Welcome.html) to create a role ARN. Take the following steps to create one: + +1. Open the **Import** page for your target cluster. + + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +2. Open the **Add New ARN** dialog. + + - If you want to import data from Amazon S3, open the **Add New ARN** dialog as follows: + + 1. Click **Import from S3**. + 2. Fill in the **File URI** field. + 3. Choose **AWS Role ARN** and click **Click here to create new one with AWS CloudFormation**. + + - If you want to export data to Amazon S3, open the **Add New ARN** dialog as follows: + + 1. Click **Export data to...** > **Amazon S3**. If your cluster has neither imported nor exported any data before, click **Click here to export data to...** > **Amazon S3** at the bottom of the page. + 2. Fill in the **Folder URI** field. + 3. Choose **AWS Role ARN** and click **Click here to create new one with AWS CloudFormation**. + +3. Create a role ARN with an AWS CloudFormation template. + + 1. In the **Add New ARN** dialog, click **AWS Console with CloudFormation Template**. + + 2. Log in to the [AWS Management Console](https://console.aws.amazon.com) and you will be redirected to the AWS CloudFormation **Quick create stack** page. + + 3. Fill in the **Role Name**. + + 4. Acknowledge to create a new role and click **Create stack** to create the role ARN. + + 5. After the CloudFormation stack is executed, you can click the **Outputs** tab and find the Role ARN value in the **Value** column. + + ![img.png](/media/tidb-cloud/serverless-external-storage/serverless-role-arn.png) + +If you have any trouble creating a role ARN with AWS CloudFormation, you can take the following steps to create one manually: + +
    +Click here to see details + +1. In the **Add New ARN** dialog described in previous instructions, click **Having trouble? Create Role ARN manually**. You will get the **TiDB Cloud Account ID** and **TiDB Cloud External ID**. + +2. In the AWS Management Console, create a managed policy for your Amazon S3 bucket. + + 1. Sign in to the [AWS Management Console](https://console.aws.amazon.com/) and open the [Amazon S3 console](https://console.aws.amazon.com/s3/). + + 2. In the **Buckets** list, choose the name of your bucket with the source data, and then click **Copy ARN** to get your S3 bucket ARN (for example, `arn:aws:s3:::tidb-cloud-source-data`). Take a note of the bucket ARN for later use. + + ![Copy bucket ARN](/media/tidb-cloud/copy-bucket-arn.png) + + 3. Open the [IAM console](https://console.aws.amazon.com/iam/), click **Policies** in the left navigation pane, and then click **Create Policy**. + + ![Create a policy](/media/tidb-cloud/aws-create-policy.png) + + 4. On the **Create policy** page, click the **JSON** tab. + + 5. Configure the policy in the policy text field according to your needs. The following is an example that you can use to export data from and import data to a TiDB Cloud Serverless cluster. + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:PutObject" + ], + "Resource": "//*" + }, + { + "Sid": "VisualEditor1", + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": "" + } + ] + } + ``` + + In the policy text field, replace the following configurations with your own values. + + - `"Resource": "//*"`. For example, + + - If your source data is stored in the root directory of the `tidb-cloud-source-data` bucket, use `"Resource": "arn:aws:s3:::tidb-cloud-source-data/*"`. + - If your source data is stored in the `mydata` directory of the bucket, use `"Resource": "arn:aws:s3:::tidb-cloud-source-data/mydata/*"`. + + Make sure that `/*` is added to the end of the directory so TiDB Cloud can access all files in this directory. + + - `"Resource": ""`, for example, `"Resource": "arn:aws:s3:::tidb-cloud-source-data"`. + + - If you have enabled AWS Key Management Service key (SSE-KMS) with customer-managed key encryption, make sure the following configuration is included in the policy. `"arn:aws:kms:ap-northeast-1:105880447796:key/c3046e91-fdfc-4f3a-acff-00597dd3801f"` is a sample KMS key of the bucket. + + ``` + { + "Sid": "AllowKMSkey", + "Effect": "Allow", + "Action": [ + "kms:Decrypt" + ], + "Resource": "arn:aws:kms:ap-northeast-1:105880447796:key/c3046e91-fdfc-4f3a-acff-00597dd3801f" + } + ``` + + - If the objects in your bucket have been copied from another encrypted bucket, the KMS key value needs to include the keys of both buckets. For example, `"Resource": ["arn:aws:kms:ap-northeast-1:105880447796:key/c3046e91-fdfc-4f3a-acff-00597dd3801f","arn:aws:kms:ap-northeast-1:495580073302:key/0d7926a7-6ecc-4bf7-a9c1-a38f0faec0cd"]`. + + 6. Click **Next**. + + 7. Set a policy name, add a tag of the policy (optional), and then click **Create policy**. + +3. In the AWS Management Console, create an access role for TiDB Cloud and get the role ARN. + + 1. In the [IAM console](https://console.aws.amazon.com/iam/), click **Roles** in the left navigation pane, and then click **Create role**. + + ![Create a role](/media/tidb-cloud/aws-create-role.png) + + 2. To create a role, fill in the following information: + + - In **Trusted entity type**, select **AWS account**. + - In **An AWS account**, select **Another AWS account**, and then paste the TiDB Cloud account ID to the **Account ID** field. + - In **Options**, click **Require external ID (Best practice when a third party will assume this role)**, and then paste the TiDB Cloud External ID to the **External ID** field. If the role is created without a Require external ID, once the configuration is done for one TiDB cluster in a project, all TiDB clusters in that project can use the same Role ARN to access your Amazon S3 bucket. If the role is created with the account ID and external ID, only the corresponding TiDB cluster can access the bucket. + + 3. Click **Next** to open the policy list, choose the policy you just created, and then click **Next**. + + 4. In **Role details**, set a name for the role, and then click **Create role** in the lower-right corner. After the role is created, the list of roles is displayed. + + 5. In the list of roles, click the name of the role that you just created to go to its summary page, and then you can get the role ARN. + + ![Copy AWS role ARN](/media/tidb-cloud/aws-role-arn.png) + +
    + +
    + +
    + +It is recommended that you use an IAM user (instead of the AWS account root user) to create an access key. + +Take the following steps to configure an access key: + +1. Create an IAM user. For more information, see [creating an IAM user](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html#id_users_create_console). + +2. Use your AWS account ID or account alias, and your IAM user name and password to sign in to [the IAM console](https://console.aws.amazon.com/iam). + +3. Create an access key. For more information, see [creating an access key for an IAM user](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey). + +> **Note:** +> +> TiDB Cloud does not store your access keys. It is recommended that you [delete the access key](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey) after the import or export is complete. + +
    +
    + +## Configure GCS access + +To allow a TiDB Serverless cluster to access your GCS bucket, you need to configure the GCS access for the bucket. You can use a service account key to configure the bucket access: + +Take the following steps to configure a service account key: + +1. On the Google Cloud [service account page](https://console.cloud.google.com/iam-admin/serviceaccounts), click **CREATE SERVICE ACCOUNT** to create a service account. For more information, see [Creating a service account](https://cloud.google.com/iam/docs/creating-managing-service-accounts). + + 1. Enter a service account name. + 2. Optional: Enter a description of the service account. + 3. Click **CREATE AND CONTINUE** to create the service account. + 4. In the `Grant this service account access to project`, choose the [IAM roles](https://cloud.google.com/iam/docs/understanding-roles) with the needed permission. For example, exporting data to a TiDB Cloud Serverless cluster needs a role with `storage.objects.create` permission. + 5. Click **Continue** to go to the next step. + 6. Optional: In the `Grant users access to this service account`, choose members that need to [attach the service account to other resources](https://cloud.google.com/iam/docs/attach-service-accounts). + 7. Click **Done** to finish creating the service account. + + ![service-account](/media/tidb-cloud/serverless-external-storage/gcs-service-account.png) + +2. Click the service account, and then click **ADD KEY** on the `KEYS` page to create a service account key. + + ![service-account-key](/media/tidb-cloud/serverless-external-storage/gcs-service-account-key.png) + +3. Choose the default `JSON` key type, and then click the **CREATE** button to download the service account key. + +## Configure Azure Blob Storage access + +To allow TiDB Serverless to access your Azure Blob container, you need to configure the Azure Blob access for the container. You can use a service SAS token to configure the container access: + +1. On the [Azure Storage account](https://portal.azure.com/#browse/Microsoft.Storage%2FStorageAccounts) page, click your storage account to which the container belongs. + +2. On your **Storage account** page, click the **Security+network**, and then click **Shared access signature**. + + ![sas-position](/media/tidb-cloud/serverless-external-storage/azure-sas-position.png) + +3. On the **Shared access signature** page, create a service SAS token with needed permissions as follows. For more information, see [Create a service SAS token](https://docs.microsoft.com/en-us/azure/storage/common/storage-sas-overview). + + 1. In the **Allowed services** section, choose the **Blob** service. + 2. In the **Allowed Resource types** section, choose **Container** and **Object**. + 3. In the **Allowed permissions** section, choose the permission as needed. For example, exporting data to a TiDB Cloud Serverless cluster needs the **Read** and **Write** permissions. + 4. Adjust **Start and expiry date/time** as needed. + 5. You can keep the default values for other settings. + + ![sas-create](/media/tidb-cloud/serverless-external-storage/azure-sas-create.png) + +4. Click **Generate SAS and connection string** to generate the SAS token. diff --git a/tidb-cloud/serverless-faqs.md b/tidb-cloud/serverless-faqs.md index 78f7cb3fbf408..4675b76fd11cf 100644 --- a/tidb-cloud/serverless-faqs.md +++ b/tidb-cloud/serverless-faqs.md @@ -1,75 +1,105 @@ --- -title: TiDB Serverless FAQs -summary: Learn about the most frequently asked questions (FAQs) relating to TiDB Serverless. +title: TiDB Cloud Serverless FAQs +summary: Learn about the most frequently asked questions (FAQs) relating to TiDB Cloud Serverless. aliases: ['/tidbcloud/serverless-tier-faqs'] --- -# TiDB Serverless FAQs +# TiDB Cloud Serverless FAQs -This document lists the most frequently asked questions about TiDB Serverless. +This document lists the most frequently asked questions about TiDB Cloud Serverless. ## General FAQs -### What is TiDB Serverless? +### What is TiDB Cloud Serverless? -TiDB Serverless offers the TiDB database with full HTAP capabilities for you and your organization. It is a fully managed, auto-scaling deployment of TiDB that lets you start using your database immediately, develop and run your application without caring about the underlying nodes, and automatically scale based on your application's workload changes. +TiDB Cloud Serverless offers the TiDB database with full HTAP capabilities for you and your organization. It is a fully managed, auto-scaling deployment of TiDB that lets you start using your database immediately, develop and run your application without caring about the underlying nodes, and automatically scale based on your application's workload changes. -### How do I get started with TiDB Serverless? +### How do I get started with TiDB Cloud Serverless? Get started with the 5-minute [TiDB Cloud Quick Start](/tidb-cloud/tidb-cloud-quickstart.md). -### How many TiDB Serverless clusters can I create in TiDB Cloud? +### How many TiDB Cloud Serverless clusters can I create in TiDB Cloud? -For each organization in TiDB Cloud, you can create a maximum of five TiDB Serverless clusters by default. To create more TiDB Serverless clusters, you need to add a credit card and set a [spending limit](/tidb-cloud/tidb-cloud-glossary.md#spending-limit) for usage. +For each organization in TiDB Cloud, you can create a maximum of five [free clusters](/tidb-cloud/select-cluster-tier.md#free-cluster-plan) by default. To create more TiDB Cloud Serverless clusters, you need to add a credit card and create [scalable clusters](/tidb-cloud/select-cluster-tier.md#scalable-cluster-plan) for the usage. -### Are all TiDB Cloud features fully supported on TiDB Serverless? +### Are all TiDB Cloud features fully supported on TiDB Cloud Serverless? -Some of TiDB Cloud features are partially supported or not supported on TiDB Serverless. For more information, see [TiDB Serverless Limitations and Quotas](/tidb-cloud/serverless-limitations.md). +Some of TiDB Cloud features are partially supported or not supported on TiDB Cloud Serverless. For more information, see [TiDB Cloud Serverless Limitations and Quotas](/tidb-cloud/serverless-limitations.md). -### When will TiDB serverless be available on cloud platforms other than AWS, such as Google Cloud or Azure? +### When will TiDB Cloud Serverless be available on cloud platforms other than AWS, such as Google Cloud or Azure? -We are actively working on expanding TiDB Serverless to other cloud platforms, including Google Cloud and Azure. However, we do not have an exact timeline for now as we currently focus on filling gaps and ensuring seamless functionality across all environments. Rest assured, we are working hard to make TiDB Serverless available on more cloud platforms, and we will keep our community updated as we progress. +We are actively working on expanding TiDB Cloud Serverless to other cloud platforms, including Google Cloud and Azure. However, we do not have an exact timeline for now as we currently focus on filling gaps and ensuring seamless functionality across all environments. Rest assured, we are working hard to make TiDB Cloud Serverless available on more cloud platforms, and we will keep our community updated as we progress. -### I created a Developer Tier cluster before TiDB Serverless was available. Can I still use my cluster? +### I created a Developer Tier cluster before TiDB Cloud Serverless was available. Can I still use my cluster? -Yes, your Developer Tier cluster has been automatically migrated to the TiDB Serverless cluster, providing you with an improved user experience without any disruptions to your prior usage. +Yes, your Developer Tier cluster has been automatically migrated to the TiDB Cloud Serverless cluster, providing you with an improved user experience without any disruptions to your prior usage. + +### What is columnar storage in TiDB Cloud Serverless? + +Columnar storage in TiDB Cloud Serverless acts as an additional replica of row-based storage, ensuring strong consistency. Unlike traditional row-based storage, which stores data in rows, columnar storage organizes data in columns, optimizing it for data analytics tasks. + +Columnar storage is a key feature that enables the Hybrid Transactional and Analytical Processing (HTAP) capabilities of TiDB by seamlessly blending transactional and analytical workloads. + +To efficiently manage columnar storage data, TiDB Cloud Serverless uses a separate elastic TiFlash engine. During query execution, the optimizer guides the cluster to automatically decide whether to retrieve data from row-based or columnar storage. + +### When should I use columnar storage in TiDB Cloud Serverless? + +Consider using columnar storage in TiDB Cloud Serverless in the following scenarios: + +- Your workload involves analytical tasks that require efficient data scanning and aggregation. +- You prioritize improved performance, especially for analytics workloads. +- You want to isolate analytical processing from transactional processing to prevent performance impact on your transactional processing (TP) workload. The separate columnar storage helps optimize these distinct workload patterns. + +In these scenarios, columnar storage can significantly improve query performance and provide a seamless experience for mixed workloads in your system. + +### How to use columnar storage in TiDB Cloud Serverless? + +Using columnar storage in TiDB Cloud Serverless is similar to using it in TiFlash. You can enable columnar storage at both the table and database levels: + +- Table level: Assign a TiFlash replica to a table to enable columnar storage for that specific table. +- Database level: Configure TiFlash replicas for all tables in a database to use columnar storage across the entire database. + +Once a TiFlash replica is set up for a table, TiDB automatically replicates data from the row-based storage to the columnar storage for that table. This ensures data consistency and optimizes performance for analytical queries. + +For more information about how to set up TiFlash replicas, see [Create TiFlash replicas](/tiflash/create-tiflash-replicas.md). ## Billing and metering FAQs ### What are Request Units? -TiDB Serverless adopts a pay-as-you-go model, meaning that you only pay for the storage space and cluster usage. In this model, all cluster activities such as SQL queries, bulk operations, and background jobs are quantified in [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit). RU is an abstract measurement for the size and intricacy of requests initiated on your cluster. For more information, see [TiDB Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details/). +TiDB Cloud Serverless adopts a pay-as-you-go model, meaning that you only pay for the storage space and cluster usage. In this model, all cluster activities such as SQL queries, bulk operations, and background jobs are quantified in [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit). RU is an abstract measurement for the size and intricacy of requests initiated on your cluster. For more information, see [TiDB Cloud Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details/). -### Is there any free plan available for TiDB Serverless? +### Is there any free plan available for TiDB Cloud Serverless? -For the first five TiDB Serverless clusters in your organization, TiDB Cloud provides a free usage quota for each of them as follows: +For the first five TiDB Cloud Serverless clusters in your organization, TiDB Cloud provides a free usage quota for each of them as follows: - Row-based storage: 5 GiB +- Columnar storage: 5 GiB - [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit): 50 million RUs per month -Usage beyond the free quota will be charged. Once the free quota of a cluster is reached, the read and write operations on this cluster will be throttled until you [increase the quota](/tidb-cloud/manage-serverless-spend-limit.md#update-spending-limit) or the usage is reset upon the start of a new month. +If you are using a scalable cluster, usage beyond the free quota will be charged. For a free cluster, once the free quota is reached, the read and write operations on this cluster will be throttled until you upgrade to a scalable cluster or the usage is reset upon the start of a new month. -For more information, see [TiDB Serverless usage quota](/tidb-cloud/select-cluster-tier.md#usage-quota). +For more information, see [TiDB Cloud Serverless usage quota](/tidb-cloud/select-cluster-tier.md#usage-quota). ### What are the limitations of the free plan? -Under the free plan, cluster performance is capped at a maximum of 10,000 RUs per second based on actual workload. Additionally, memory allocation per query is limited to 256 MiB. To maximize cluster performance, you can choose to enable the commercial offering by [increasing your spending limit](/tidb-cloud/manage-serverless-spend-limit.md#update-spending-limit). +Under the free plan, cluster performance is limited due to non-scalable resources. This results in a restriction on memory allocation per query to 256 MiB and might cause observable bottlenecks in request units (RUs) per second. To maximize cluster performance and avoid these limitations, you can upgrade to a [scalable cluster](/tidb-cloud/select-cluster-tier.md#scalable-cluster-plan). ### How can I estimate the number of RUs required by my workloads and plan my monthly budget? To get the RU consumption of individual SQL statements, you can use the [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md#ru-request-unit-consumption) SQL statement. However, it is important to note that the RUs usage returned in `EXPLAIN ANALYZE` does not incorporate egress RUs, as egress usage is measured separately in the gateway, which is unknown to the TiDB server. -To get the RUs and storage used by your cluster, view the **Usage this month** pane on your cluster overview page. With your past resource usage data and real-time resource usage in this pane, you can track your cluster's resource consumption and estimate a reasonable spending limit. If the free quota cannot meet your requirement, you can edit the spending limit easily. For more information, see [Manage Spending Limit for TiDB Serverless clusters](/tidb-cloud/manage-serverless-spend-limit.md). +To get the RUs and storage used by your cluster, view the **Usage this month** pane on your cluster overview page. With your past resource usage data and real-time resource usage in this pane, you can track your cluster's resource consumption and estimate a reasonable spending limit. If the free quota cannot meet your requirement, you can upgrade to a [scalable cluster](/tidb-cloud/select-cluster-tier.md#scalable-cluster-plan) and edit the spending limit. For more information, see [TiDB Cloud Serverless usage quota](/tidb-cloud/select-cluster-tier.md#usage-quota). ### How can I optimize my workload to minimize the number of RUs consumed? -Ensure that your queries have been carefully optimized for optimal performance by following the guidelines in [Optimizing SQL Performance](/develop/dev-guide-optimize-sql-overview.md). In addition, minimizing the amount of egress traffic is also crucial for reducing RUs consumption. To achieve this, it is recommended to return only the necessary columns and rows in your query, which in turn helps reduce network egress traffic. This can be achieved by carefully selecting and filtering the columns and rows to be returned, thereby optimizing network utilization. +Ensure that your queries have been carefully optimized for optimal performance by following the guidelines in [Optimizing SQL Performance](/develop/dev-guide-optimize-sql-overview.md). To identify the SQL statements that consume the most RUs, navigate to **Diagnosis > SQL Statements** on your cluster overview page, where you can observe SQL execution and view the top statements sorted by **Total RU** or **Mean RU**. For more information, see [Statement Analysis](/tidb-cloud/tune-performance.md#statement-analysis). In addition, minimizing the amount of egress traffic is also crucial for reducing RUs consumption. To achieve this, it is recommended to return only the necessary columns and rows in your query, which in turn helps reduce network egress traffic. This can be achieved by carefully selecting and filtering the columns and rows to be returned, thereby optimizing network utilization. -### How storage is metered for TiDB Serverless? +### How storage is metered for TiDB Cloud Serverless? -The storage is metered based on the amount of data stored in a TiDB Serverless cluster, measured in GiB per month. It is calculated by multiplying the total size of all the tables and indexes (excluding data compression or replicas) with the number of hours the data is stored in that month. +The storage is metered based on the amount of data stored in a TiDB Cloud Serverless cluster, measured in GiB per month. It is calculated by multiplying the total size of all the tables and indexes (excluding data compression or replicas) with the number of hours the data is stored in that month. ### Why does the storage usage size remain unchanged after dropping a table or database immediately? @@ -85,21 +115,39 @@ A spike in RU usage can occur due to necessary background jobs in TiDB. These jo ### What happens when my cluster exhausts its free quota or exceeds its spending limit? -Once a cluster reaches its free quota or spending limit, the cluster will enforce throttling measures on its read and write operations. These operations will be limited until the quota is increased or the usage is reset at the start of a new month. For more information, see [TiDB Serverless Limitations and Quotas](/tidb-cloud/serverless-limitations.md#usage-quota). +Once a cluster reaches its free quota or spending limit, the cluster immediately denies any new connection attempts until the quota is increased or the usage is reset at the start of a new month. Existing connections established before reaching the quota will remain active but will experience throttling. For more information, see [TiDB Cloud Serverless Limitations and Quotas](/tidb-cloud/serverless-limitations.md#usage-quota). + +### Why do I observe spikes in RU usage while importing data? + +During the data import process of a TiDB Cloud Serverless cluster, RU consumption occurs only when the data is successfully imported, which leads to spikes in RU usage. + +### What costs are involved when using columnar storage in TiDB Cloud Serverless? + +The pricing for columnar storage in TiDB Cloud Serverless is similar to that for row-based storage. When you use columnar storage, an additional replica is created to store your data (without indexes). The replication of data from row-based to columnar storage does not incur extra charges. + +For detailed pricing information, see [TiDB Cloud Serverless pricing details](https://www.pingcap.com/tidb-serverless-pricing-details/). + +### Is using columnar storage more expensive? + +Columnar storage in TiDB Cloud Serverless incurs additional costs due to the extra replica, which requires more storage and resources for data replication. However, columnar storage becomes more cost-effective when running analytical queries. + +According to the TPC-H benchmark test, the cost of running analytic queries on columnar storage is about one-third of the cost when using row-based storage. + +Therefore, while there might be an initial cost due to the extra replica, the reduced computational costs during analytics can make it more cost-effective for specific use cases. Especially for users with analytical demands, columnar storage can significantly reduce costs, offering considerable cost savings opportunities. ## Security FAQs -### Is my TiDB Serverless shared or dedicated? +### Is my TiDB Cloud Serverless shared or dedicated? -The serverless technology is designed for multi-tenancy and the resources used by all clusters are shared. To get managed TiDB service with isolated infrastructure and resources, you can upgrade it to [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated). +The serverless technology is designed for multi-tenancy and the resources used by all clusters are shared. To get managed TiDB service with isolated infrastructure and resources, you can upgrade it to [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). -### How does TiDB Serverless ensure security? +### How does TiDB Cloud Serverless ensure security? -- Your connections are encrypted by Transport Layer Security (TLS). For more information about using TLS to connect to TiDB Serverless, see [TLS Connection to TiDB Serverless](/tidb-cloud/secure-connections-to-serverless-clusters.md). -- All persisted data on TiDB Serverless is encrypted-at-rest using the tool of the cloud provider that your cluster is running in. +- Your connections are encrypted by Transport Layer Security (TLS). For more information about using TLS to connect to TiDB Cloud Serverless, see [TLS Connection to TiDB Cloud Serverless](/tidb-cloud/secure-connections-to-serverless-clusters.md). +- All persisted data on TiDB Cloud Serverless is encrypted-at-rest using the tool of the cloud provider that your cluster is running in. ## Maintenance FAQ ### Can I upgrade the version of TiDB that my cluster is running on? -No. TiDB Serverless clusters are upgraded automatically as we roll out new TiDB versions on TiDB Cloud. You can see what version of TiDB your cluster is running in the [TiDB Cloud console](https://tidbcloud.com/console/clusters) or in the latest [release note](https://docs.pingcap.com/tidbcloud/tidb-cloud-release-notes). Alternatively, you can also connect to your cluster and use `SELECT version()` or `SELECT tidb_version()` to check the TiDB version. +No. TiDB Cloud Serverless clusters are upgraded automatically as we roll out new TiDB versions on TiDB Cloud. You can see what version of TiDB your cluster is running in the [TiDB Cloud console](https://tidbcloud.com/console/clusters) or in the latest [release note](https://docs.pingcap.com/tidbcloud/tidb-cloud-release-notes). Alternatively, you can also connect to your cluster and use `SELECT version()` or `SELECT tidb_version()` to check the TiDB version. diff --git a/tidb-cloud/serverless-high-availability.md b/tidb-cloud/serverless-high-availability.md new file mode 100644 index 0000000000000..e288e9329f771 --- /dev/null +++ b/tidb-cloud/serverless-high-availability.md @@ -0,0 +1,114 @@ +--- +title: High Availability in TiDB Cloud Serverless +summary: Learn about the high availability architecture of TiDB Cloud Serverless. Discover Zonal and Regional High Availability options, automated backups, failover processes, and how TiDB ensures data durability and business continuity. +--- + +# High Availability in TiDB Cloud Serverless + +TiDB Cloud Serverless is designed with robust mechanisms to maintain high availability and data durability by default, preventing single points of failure and ensuring continuous service even in the face of disruptions. As a fully managed service based on the battle-tested TiDB Open Source product, it inherits TiDB's core high availability (HA) features and augments them with additional cloud-native capabilities. + +## Overview + +TiDB ensures high availability and data durability using the Raft consensus algorithm. This algorithm consistently replicates data changes across multiple nodes, allowing TiDB to handle read and write requests even in the event of node failures or network partitions. This approach provides both high data durability and fault tolerance. + +TiDB Cloud Serverless extends these capabilities with two types of high availability to meet different operational requirements: + +- **Zonal high availability (default)**: This option places all nodes within a single availability zone, reducing network latency. It ensures high availability without requiring application-level redundancy across zones, making it suitable for applications that prioritize low latency within a single zone. Zonal high availability is available in all regions that support TiDB Cloud Serverless. For more information, see [Zonal high availability architecture](#zonal-high-availability-architecture). + +- **Regional high availability (beta)**: This option distributes nodes across multiple availability zones, offering maximum infrastructure isolation and redundancy. It provides the highest level of availability but requires application-level redundancy across zones. It is recommended to choose this option if you need maximum availability protection against infrastructure failures within a zone. Note that it increases latency and might incur cross-zone data transfer fees. This feature is available in selected regions with multi-availability zone support and can only be enabled during cluster creation. For more information, see [Regional high availability architecture](#regional-high-availability-architecture). + +## Zonal high availability architecture + +> **Note:** +> +> Zonal high availability is the default option and is available in all AWS regions that support TiDB Cloud Serverless. + +When you create a cluster with the default zonal high availability, all components, including Gateway, TiDB, TiKV, and TiFlash compute/write nodes, run in the same availability zone. The placement of these components in the data plane offer infrastructure redundancy with virtual machine pools, which minimizes failover time and network latency due to colocation. + +![TiDB Cloud Serverless zonal high availability](/media/tidb-cloud/serverless-zonal-high-avaliability-aws.png) + +In zonal high availability architecture: + +- The Placement Driver (PD) is deployed across multiple availability zones, ensuring high availability by replicating data redundantly across zones. +- Data is replicated across TiKV servers and TiFlash write nodes within the local availability zone. +- TiDB servers and TiFlash compute nodes read from and write to TiKV and TiFlash write nodes, which are safeguarded by storage-level replication. + +### Failover process + +TiDB Cloud Serverless ensures a transparent failover process for your applications. During a failover: + +- A new replica is created to replace the failed one. + +- Servers providing storage services recover local caches from persisted data on Amazon S3, restoring the system to a consistent state with the replicas. + +In the storage layer, persisted data is regularly pushed to Amazon S3 for high durability. Moreover, immediate updates are not only replicated across multiple TiKV servers but also stored on the EBS of each server, which further replicates the data for additional durability. TiDB automatically resolves issues by backing off and retrying in milliseconds, ensuring the failover process remains seamless for client applications. + +The gateway and computing layers are stateless, so failover involves restarting them elsewhere immediately. Applications should implement retry logic for their connections. While the zonal setup provides high availability, it cannot handle an entire zone failure. If the zone becomes unavailable, downtime will occur until the zone and its dependent services are restored. + +## Regional high availability architecture + +When you create a cluster with regional high availability, critical OLTP (Online Transactional Processing) workload components, such as PD and TiKV, are deployed across multiple availability zones to ensure redundant replication and maximizing availability. During normal operations, components like Gateway, TiDB, and TiFlash compute/write nodes are hosted in the primary availability zone. These components in data plane offer infrastructure redundancy through virtual machine pools, which minimizes failover time and network latency due to colocation. + +> **Note:** +> +> - Regional high availability is currently in beta and only available in the AWS Tokyo (`ap-northeast-1`) region. +> - You can enable regional high availability only during cluster creation. + +![TiDB Cloud Serverless regional high availability](/media/tidb-cloud/serverless-regional-high-avaliability-aws.png) + +In regional high availability architecture: + +- The Placement Driver (PD) and TiKV are deployed across multiple availability zones, and data is always replicated redundantly across zones to ensure the highest level of availability. +- Data is replicated across TiFlash write nodes within the primary availability zone. +- TiDB servers and TiFlash compute nodes read from and write to these TiKV and TiFlash write nodes, which are safeguarded by storage-level replication. + +### Failover process + +In the rare event of a primary zone failure scenario, which could be caused by a natural disaster, configuration change, software issue, or hardware failure, critical OLTP workload components, including Gateway and TiDB, are automatically launched in the standby availability zone. Traffic is automatically redirected to the standby zone to ensure swift recovery and maintain business continuity. + +TiDB Cloud Serverless minimizes service disruption and ensures business continuity during a primary zone failure by performing the following actions: + +- Automatically create new replicas of Gateway and TiDB in the standby availability zone. +- Use the elastic load balancer to detect active gateway replicas in the standby availability zone and redirect OLTP traffic from the failed primary zone. + +In addition to providing high availability through TiKV replication, TiKV instances are deployed and configured to place each data replica in a different availability zone. The system remains available as long as two availability zones are operating normally. For high durability, data persistence is ensured by regularly backing up data to S3. Even if two zones fail, data stored in S3 remains accessible and recoverable. + +Applications are unaffected by failures in non-primary zones and remain unaware of such events. During a primary zone failure, Gateway and TiDB are launched in the standby availability zone to handle workloads. Ensure that your applications implement retry logic to redirect new requests to active servers in the standby availability zone. + +## Automatic backups and durability + +Database backups are essential for business continuity and disaster recovery, helping to protect your data from corruption or accidental deletion. With backups, you can restore your database to a specific point in time within the retention period, minimizing data loss and downtime. + +TiDB Cloud Serverless provides robust automated backup mechanisms to ensure continuous data protection: + +- **Daily full backups**: A full backup of your database is created once a day, capturing the entire database state. +- **Continuous transaction log backups**: Transaction logs are backed up continuously, approximately every 5 minutes, though the exact frequency depends on database activity. + +These automated backups enable you to restore your database either from a full backup or from a specific point in time by combining full backups with continuous transaction logs. This flexibility ensures that you can recover your database to a precise point just before an incident occurs. + +> **Note:** +> +> Automatic backups, including snapshot-based and continuous backups for Point-in-Time Recovery (PITR), are performed on Amazon S3, which provides regional-level high durability. + +## Impact on sessions during failures + +During a failure, ongoing transactions on the failed server might be interrupted. Although failover is transparent to applications, you must implement logic to handle recoverable failures during active transactions. Different failure scenarios are handled as follows: + +- **TiDB failures**: If a TiDB instance fails, client connections are unaffected because TiDB Cloud Serverless automatically reroutes traffic through the gateway. While transactions on the failed TiDB instance might be interrupted, the system ensures that committed data is preserved, and new transactions are handled by another available TiDB instance. +- **Gateway failures**: If the Gateway fails, client connections are disrupted. However, TiDB Cloud Serverless gateways are stateless and can restart immediately in a new zone or server. Traffic is automatically redirected to the new gateway, minimizing downtime. + +It is recommended to implement retry logic in your application to handle recoverable failures. For implementation details, refer to your driver or ORM documentation (for example, [JDBC](https://dev.mysql.com/doc/connector-j/en/connector-j-config-failover.html)). + +## RTO and RPO + +When creating your business continuity plan, consider these two key metrics: + +- Recovery Time Objective (RTO): The maximum acceptable time it takes for the application to fully recover after a disruptive event. +- Recovery Point Objective (RPO): The maximum acceptable time interval of recent data updates that the application can tolerate losing during recovery from an unplanned disruptive event. + +The following table compares the RTO and RPO for each high availability option: + +| High availability architecture | RTO (downtime) | RPO (data loss) | +|--------------------------------|-------------------------------|-----------------| +| Zonal high availability | Near 0 seconds | 0 | +| Regional high availability | Typically less than 600 seconds | 0 | diff --git a/tidb-cloud/serverless-limitations.md b/tidb-cloud/serverless-limitations.md index 62f16db31c6a6..9a33b826b02f8 100644 --- a/tidb-cloud/serverless-limitations.md +++ b/tidb-cloud/serverless-limitations.md @@ -1,16 +1,16 @@ --- -title: TiDB Serverless Limitations and Quotas -summary: Learn about the limitations of TiDB Serverless. +title: TiDB Cloud Serverless Limitations and Quotas +summary: Learn about the limitations of TiDB Cloud Serverless. aliases: ['/tidbcloud/serverless-tier-limitations'] --- -# TiDB Serverless Limitations and Quotas +# TiDB Cloud Serverless Limitations and Quotas -TiDB Serverless works with almost all workloads that TiDB supports, but there are some feature differences between TiDB Self-Hosted or TiDB Dedicated clusters and TiDB Serverless clusters. This document describes the limitations of TiDB Serverless. +TiDB Cloud Serverless works with almost all workloads that TiDB supports, but there are some feature differences between TiDB Self-Managed or TiDB Cloud Dedicated clusters and TiDB Cloud Serverless clusters. This document describes the limitations of TiDB Cloud Serverless. -We are constantly filling in the feature gaps between TiDB Serverless and TiDB Dedicated. If you require these features or capabilities in the gap, use [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) or [contact us](https://www.pingcap.com/contact-us/?from=en) for a feature request. +We are constantly filling in the feature gaps between TiDB Cloud Serverless and TiDB Cloud Dedicated. If you require these features or capabilities in the gap, use [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) or [contact us](https://www.pingcap.com/contact-us/?from=en) for a feature request. ## Limitations @@ -20,12 +20,12 @@ We are constantly filling in the feature gaps between TiDB Serverless and TiDB D ### Connection -- Only [Public Endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md) and [Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md) can be used. You cannot use [VPC Peering](/tidb-cloud/set-up-vpc-peering-connections.md) to connect to TiDB Serverless clusters.  +- Only [Public Endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md) and [Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md) can be used. You cannot use [VPC Peering](/tidb-cloud/set-up-vpc-peering-connections.md) to connect to TiDB Cloud Serverless clusters.  - No [IP Access list](/tidb-cloud/configure-ip-access-list.md) support. ### Encryption -- Data persisted in your TiDB Serverless cluster is encrypted using the encryption tool provided by the cloud provider that manages your cluster. However, TiDB Serverless does not provide any additional optional measures for protecting data at-rest on disks beyond infrastructure-level encryption. +- Data persisted in your TiDB Cloud Serverless cluster is encrypted using the encryption tool provided by the cloud provider that manages your cluster. However, TiDB Cloud Serverless does not provide any additional optional measures for protecting data at-rest on disks beyond infrastructure-level encryption. - Using [customer-managed encryption keys (CMEK)](/tidb-cloud/tidb-cloud-encrypt-cmek.md) is currently unavailable. ### Maintenance window @@ -38,38 +38,41 @@ We are constantly filling in the feature gaps between TiDB Serverless and TiDB D - [Built-in Alerting](/tidb-cloud/monitor-built-in-alerting.md) is currently unavailable. - [Key Visualizer](/tidb-cloud/tune-performance.md#key-visualizer) is currently unavailable. - [Index Insight](/tidb-cloud/tune-performance.md#index-insight-beta) is currently unavailable. -- [Cluster Events](/tidb-cloud/tidb-cloud-events.md) is currently unavailable. ### Self-service upgrades -- TiDB Serverless is a fully managed deployment of TiDB. Major and minor version upgrades of TiDB Serverless are handled by TiDB Cloud and therefore cannot be initiated by users. +- TiDB Cloud Serverless is a fully managed deployment of TiDB. Major and minor version upgrades of TiDB Cloud Serverless are handled by TiDB Cloud and therefore cannot be initiated by users. ### Stream data -- [Changefeed](/tidb-cloud/changefeed-overview.md) is not supported for TiDB Serverless currently. -- [Data Migration](/tidb-cloud/migrate-from-mysql-using-data-migration.md) is not supported for TiDB Serverless currently. +- [Changefeed](/tidb-cloud/changefeed-overview.md) is not supported for TiDB Cloud Serverless currently. +- [Data Migration](/tidb-cloud/migrate-from-mysql-using-data-migration.md) is not supported for TiDB Cloud Serverless currently. + +### Time to live (TTL) + +- In TiDB Cloud Serverless, the [`TTL_JOB_INTERVAL`](/time-to-live.md#ttl-job) attribute for a table is fixed at `15m` and cannot be modified. This means that TiDB Cloud Serverless schedules a background job every 15 minutes to clean up expired data. ### Others -- [Time to live (TTL)](/time-to-live.md) is currently unavailable. - Transaction can not last longer than 30 minutes. - For more details about SQL limitations, refer to [Limited SQL Features](/tidb-cloud/limited-sql-features.md). ## Usage quota -For each organization in TiDB Cloud, you can create a maximum of five TiDB Serverless clusters by default. To create more TiDB Serverless clusters, you need to add a credit card and set a [spending limit](/tidb-cloud/tidb-cloud-glossary.md#spending-limit) for the usage. +For each organization in TiDB Cloud, you can create a maximum of five [free clusters](/tidb-cloud/select-cluster-tier.md#free-cluster-plan) by default. To create more TiDB Cloud Serverless clusters, you need to add a credit card and create [scalable clusters](/tidb-cloud/select-cluster-tier.md#scalable-cluster-plan) for the usage. -For the first five TiDB Serverless clusters in your organization, TiDB Cloud provides a free usage quota for each of them as follows: +For the first five TiDB Cloud Serverless clusters in your organization, whether they are free or scalable, TiDB Cloud provides a free usage quota for each of them as follows: - Row-based storage: 5 GiB +- Columnar storage: 5 GiB - [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit): 50 million RUs per month The Request Unit (RU) is a unit of measurement used to track the resource consumption of a query or transaction. It is a metric that allows you to estimate the computational resources required to process a specific request in the database. The request unit is also the billing unit for TiDB Cloud Serverless service. -Once the free quota of a cluster is reached, the read and write operations on this cluster will be throttled until you [increase the quota](/tidb-cloud/manage-serverless-spend-limit.md#update-spending-limit) or the usage is reset upon the start of a new month. +Once a cluster reaches its usage quota, it immediately denies any new connection attempts until you [increase the quota](/tidb-cloud/manage-serverless-spend-limit.md#update-spending-limit) or the usage is reset upon the start of a new month. Existing connections established before reaching the quota will remain active but will experience throttling. -To learn more about the RU consumption of different resources (including read, write, SQL CPU, and network egress), the pricing details, and the throttled information, see [TiDB Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details). +To learn more about the RU consumption of different resources (including read, write, SQL CPU, and network egress), the pricing details, and the throttled information, see [TiDB Cloud Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details). -If you want to create a TiDB Serverless cluster with an additional quota, you can edit the spending limit on the cluster creation page. For more information, see [Create a TiDB Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md). +If you want to create a TiDB Cloud Serverless cluster with an additional quota, you can select the scalable cluster plan and edit the spending limit on the cluster creation page. For more information, see [Create a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md). -After creating a TiDB Serverless, you can still check and edit the spending limit on your cluster overview page. For more information, see [Manage Spending Limit for TiDB Serverless Clusters](/tidb-cloud/manage-serverless-spend-limit.md). +After creating a TiDB Cloud Serverless cluster, you can still check and edit the spending limit on your cluster overview page. For more information, see [Manage Spending Limit for TiDB Cloud Serverless Clusters](/tidb-cloud/manage-serverless-spend-limit.md). diff --git a/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md b/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md index ad728d54d0071..2ae40e9d64ff9 100644 --- a/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md +++ b/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md @@ -1,16 +1,16 @@ --- -title: Connect to a TiDB Dedicated Cluster via Google Cloud Private Service Connect +title: Connect to a TiDB Cloud Dedicated Cluster via Google Cloud Private Service Connect summary: Learn how to connect to your TiDB Cloud cluster via Google Cloud Private Service Connect. --- -# Connect to a TiDB Dedicated Cluster via Google Cloud Private Service Connect +# Connect to a TiDB Cloud Dedicated Cluster via Google Cloud Private Service Connect -This document describes how to connect to your TiDB Dedicated cluster via Google Cloud Private Service Connect. Google Cloud Private Service Connect is a private endpoint service provided by Google Cloud. +This document describes how to connect to your TiDB Cloud Dedicated cluster via Google Cloud Private Service Connect. Google Cloud Private Service Connect is a private endpoint service provided by Google Cloud. > **Tip:** > -> - To learn how to connect to a TiDB Dedicated cluster via private endpoint with AWS, see [Connect to TiDB Dedicated via Private Endpoint with AWS](/tidb-cloud/set-up-private-endpoint-connections.md). -> - To learn how to connect to a TiDB Serverless cluster via private endpoint, see [Connect to TiDB Serverless via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md). +> - To learn how to connect to a TiDB Cloud Dedicated cluster via private endpoint with AWS, see [Connect to TiDB Cloud Dedicated via Private Endpoint with AWS](/tidb-cloud/set-up-private-endpoint-connections.md). +> - To learn how to connect to a TiDB Cloud Serverless cluster via private endpoint, see [Connect to TiDB Cloud Serverless via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md). TiDB Cloud supports highly secure and one-way access to the TiDB Cloud service hosted in a Google Cloud VPC via the [Private Service Connect](https://cloud.google.com/vpc/docs/private-service-connect). You can create an endpoint and use it to connect to the TiDB Cloud service . @@ -27,11 +27,11 @@ For more detailed definitions of the private endpoint and endpoint service, see ## Restrictions -- This feature is applicable to TiDB Dedicated clusters created after April 13, 2023. For older clusters, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md) for assistance. +- This feature is applicable to TiDB Cloud Dedicated clusters created after April 13, 2023. For older clusters, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md) for assistance. - Only the `Organization Owner` and `Project Owner` roles can create Google Cloud Private Service Connect endpoints. - Each TiDB cluster can handle connections from up to 10 endpoints. - Each Google Cloud project can have up to 10 endpoints connecting to a TiDB Cluster. -- You can create up to 8 TiDB Dedicated clusters hosted on Google Cloud in a project with the endpoint service configured. +- You can create up to 8 TiDB Cloud Dedicated clusters hosted on Google Cloud in a project with the endpoint service configured. - The private endpoint and the TiDB cluster to be connected must be located in the same region. - Egress firewall rules must permit traffic to the internal IP address of the endpoint. The [implied allow egress firewall rule](https://cloud.google.com/firewall/docs/firewalls#default_firewall_rules) permits egress to any destination IP address. - If you have created egress deny firewall rules in your VPC network, or if you have created hierarchical firewall policies that modify the implied allowed egress behavior, access to the endpoint might be affected. In this case, you need to create a specific egress allow firewall rule or policy to permit traffic to the internal destination IP address of the endpoint. @@ -43,10 +43,10 @@ In most scenarios, it is recommended that you use private endpoint connection ov ## Set up a private endpoint with Google Cloud Private Service Connect -To connect to your TiDB Dedicated cluster via a private endpoint, complete the [prerequisites](#prerequisites) and follow these steps: +To connect to your TiDB Cloud Dedicated cluster via a private endpoint, complete the [prerequisites](#prerequisites) and follow these steps: -1. [Choose a TiDB cluster](#step-1-choose-a-tidb-cluster) -2. [Provide the information for creating an endpoint](#step-2-provide-the-information-for-creating-an-endpoint) +1. [Select a TiDB cluster](#step-1-select-a-tidb-cluster) +2. [Create a Google Cloud private endpoint](#step-2-create-a-google-cloud-private-endpoint) 3. [Accept endpoint access](#step-3-accept-endpoint-access) 4. [Connect to your TiDB cluster](#step-4-connect-to-your-tidb-cluster) @@ -70,25 +70,24 @@ Before you begin to create an endpoint: - [Compute Network Admin](https://cloud.google.com/iam/docs/understanding-roles#compute.networkAdmin) (roles/compute.networkAdmin) - [Service Directory Editor](https://cloud.google.com/iam/docs/understanding-roles#servicedirectory.editor) (roles/servicedirectory.editor) -Perform the following steps to go to the **Google Cloud Private Endpoint** page: +### Step 1. Select a TiDB cluster -1. Log in to the [TiDB Cloud console](https://tidbcloud.com). -2. Click in the lower-left corner, switch to the target project if you have multiple projects, and then click **Project Settings**. -3. On the **Project Settings** page of your project, click **Network Access** in the left navigation pane, and click the **Private Endpoint** tab. -4. Click **Create Private Endpoint** in the upper-right corner, and then select **Google Cloud Private Endpoint**. +1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click the name of your target TiDB cluster to go to its overview page. You can select a cluster with any of the following statuses: -### Step 1. Choose a TiDB cluster + - **Available** + - **Restoring** + - **Modifying** + - **Importing** -Click the drop-down list and choose an available TiDB Dedicated cluster. +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -You can select a cluster with any of the following statuses: +3. In the **Connection Type** drop-down list, select **Private Endpoint**, and then click **Create Private Endpoint Connection**. -- **Available** -- **Restoring** -- **Modifying** -- **Importing** + > **Note:** + > + > If you have already created a private endpoint connection, the active endpoint will appear in the connection dialog. To create additional private endpoint connections, navigate to the **Networking** page in the left navigation pane. -### Step 2. Provide the information for creating an endpoint +### Step 2. Create a Google Cloud private endpoint 1. Provide the following information to generate the command for private endpoint creation: - **Google Cloud Project ID**: the Project ID associated with your Google Cloud account. You can find the ID on the [Google Cloud **Dashboard** page](https://console.cloud.google.com/home/dashboard). @@ -96,8 +95,8 @@ You can select a cluster with any of the following statuses: - **Google Cloud Subnet Name**: the name of the subnet in the specified VPC. You can find it on the **VPC network details** page. - **Private Service Connect Endpoint Name**: enter a unique name for the private endpoint that will be created. 2. After entering the information, click **Generate Command**. -3. Copy the command. -4. Go to [Google Cloud Shell](https://console.cloud.google.com/home/dashboard) to execute the command. +3. Copy the generated command. +4. Open [Google Cloud Shell](https://console.cloud.google.com/home/dashboard) and execute the command to create the private endpoint. ### Step 3. Accept endpoint access @@ -107,11 +106,11 @@ If you see an error `not received connection request from endpoint`, make sure t ### Step 4. Connect to your TiDB cluster -After you have accepted the endpoint connection, take the following steps to connect to your TiDB cluster: +After you have accepted the private endpoint connection, you are redirected back to the connection dialog. -1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click **...** in the **Action** column. -2. Click **Connect**. A connection dialog is displayed. -3. Select the **Private Endpoint** tab. The private endpoint you just created is displayed. Copy the command to connect to the TiDB cluster. +1. Wait for the private endpoint connection status to change from **System Checking** to **Active** (approximately 5 minutes). +2. In the **Connect With** drop-down list, select your preferred connection method. The corresponding connection string is displayed at the bottom of the dialog. +3. Connect to your cluster with the connection string. ### Private endpoint status reference @@ -133,7 +132,7 @@ The possible statuses of a private endpoint service are explained as follows: ### TiDB Cloud fails to create an endpoint service. What should I do? -The endpoint service is created automatically after you open the **Create Google Cloud Private Endpoint** page and choose the TiDB cluster. If it shows as failed or remains in the **Creating** state for a long time, submit a [support ticket](/tidb-cloud/tidb-cloud-support.md) for assistance. +The endpoint service is created automatically after you open the **Create Google Cloud Private Endpoint Connection** page and choose the TiDB cluster. If it shows as failed or remains in the **Creating** state for a long time, submit a [support ticket](/tidb-cloud/tidb-cloud-support.md) for assistance. ### Fail to create an endpoint in Google Cloud. What should I do? @@ -147,6 +146,6 @@ If you have already executed the command to create a private endpoint in Google ### Why can't I see the endpoints generated by directly copying the service attachment in the TiDB Cloud console? -In the TiDB Cloud console, you can only view endpoints that are created through the command generated on the **Create Google Cloud Private Endpoint** page. +In the TiDB Cloud console, you can only view endpoints that are created through the command generated on the **Create Google Cloud Private Endpoint Connection** page. However, endpoints generated by directly copying the service attachment (that is, not created through the command generated in the TiDB Cloud console) are not displayed in the TiDB Cloud console. diff --git a/tidb-cloud/set-up-private-endpoint-connections-serverless.md b/tidb-cloud/set-up-private-endpoint-connections-serverless.md index fc067b66d3baf..a2dddbcc98b2d 100644 --- a/tidb-cloud/set-up-private-endpoint-connections-serverless.md +++ b/tidb-cloud/set-up-private-endpoint-connections-serverless.md @@ -1,16 +1,16 @@ --- -title: Connect to TiDB Serverless via Private Endpoint +title: Connect to TiDB Cloud Serverless via Private Endpoint summary: Learn how to connect to your TiDB Cloud cluster via private endpoint. --- -# Connect to TiDB Serverless via Private Endpoint +# Connect to TiDB Cloud Serverless via Private Endpoint -This document describes how to connect to your TiDB Serverless cluster via private endpoint. +This document describes how to connect to your TiDB Cloud Serverless cluster via private endpoint. > **Tip:** > -> To learn how to connect to a TiDB Dedicated cluster via private endpoint with AWS, see [Connect to TiDB Dedicated via Private Endpoint with AWS](/tidb-cloud/set-up-private-endpoint-connections.md). -> To learn how to connect to a TiDB Dedicated cluster via private endpoint with Google Cloud, see [Connect to TiDB Dedicated via Private Service Connect with Google Cloud](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md). +> To learn how to connect to a TiDB Cloud Dedicated cluster via private endpoint with AWS, see [Connect to TiDB Cloud Dedicated via Private Endpoint with AWS](/tidb-cloud/set-up-private-endpoint-connections.md). +> To learn how to connect to a TiDB Cloud Dedicated cluster via private endpoint with Google Cloud, see [Connect to TiDB Cloud Dedicated via Private Service Connect with Google Cloud](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md). TiDB Cloud supports highly secure and one-way access to the TiDB Cloud service hosted in an AWS VPC via the [AWS PrivateLink](https://aws.amazon.com/privatelink/?privatelink-blogs.sort-by=item.additionalFields.createdDate&privatelink-blogs.sort-order=desc), as if the service were in your own VPC. A private endpoint is exposed in your VPC and you can create a connection to the TiDB Cloud service via the endpoint with permission. @@ -27,12 +27,12 @@ For more detailed definitions of the private endpoint and endpoint service, see ## Restrictions -- Currently, TiDB Cloud supports private endpoint connection to TiDB Serverless only when the endpoint service is hosted in AWS. If the service is hosted in Google Cloud, the private endpoint is not applicable. +- Currently, TiDB Cloud supports private endpoint connection to TiDB Cloud Serverless only when the endpoint service is hosted in AWS. If the service is hosted in Google Cloud, the private endpoint is not applicable. - Private endpoint connection across regions is not supported. ## Set up a private endpoint with AWS -To connect to your TiDB Serverless cluster via a private endpoint, follow these steps: +To connect to your TiDB Cloud Serverless cluster via a private endpoint, follow these steps: 1. [Choose a TiDB cluster](#step-1-choose-a-tidb-cluster) 2. [Create an AWS interface endpoint](#step-2-create-an-aws-interface-endpoint) @@ -40,14 +40,14 @@ To connect to your TiDB Serverless cluster via a private endpoint, follow these ### Step 1. Choose a TiDB cluster -1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click the name of your target TiDB Serverless cluster to go to its overview page. +1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click the name of your target TiDB Cloud Serverless cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. In the **Endpoint Type** drop-down list, select **Private**. +3. In the **Connection Type** drop-down list, select **Private Endpoint**. 4. Take a note of **Service Name**, **Availability Zone ID**, and **Region ID**. > **Note:** > - > You only need to create one private endpoint per AWS region, which can be shared by all TiDB Serverless clusters located in the same region. + > You only need to create one private endpoint per AWS region, which can be shared by all TiDB Cloud Serverless clusters located in the same region. ### Step 2. Create an AWS interface endpoint @@ -56,7 +56,7 @@ To connect to your TiDB Serverless cluster via a private endpoint, follow these To use the AWS Management Console to create a VPC interface endpoint, perform the following steps: -1. Sign in to the [AWS Management Console](https://aws.amazon.com/console/) and open the Amazon VPC console at . +1. Sign in to the [AWS Management Console](https://aws.amazon.com/console/) and open the Amazon VPC console at [https://console.aws.amazon.com/vpc/](https://console.aws.amazon.com/vpc/). 2. Click **Endpoints** in the navigation pane, and then click **Create Endpoint** in the upper-right corner. The **Create endpoint** page is displayed. @@ -103,7 +103,7 @@ After you have created the interface endpoint, go back to the TiDB Cloud console 1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. In the **Endpoint Type** drop-down list, select **Private**. +3. In the **Connection Type** drop-down list, select **Private Endpoint**. 4. In the **Connect With** drop-down list, select your preferred connection method. The corresponding connection string is displayed at the bottom of the dialog. 5. Connect to your cluster with the connection string. diff --git a/tidb-cloud/set-up-private-endpoint-connections.md b/tidb-cloud/set-up-private-endpoint-connections.md index e7ae14973bdf9..9d16744dc1eb0 100644 --- a/tidb-cloud/set-up-private-endpoint-connections.md +++ b/tidb-cloud/set-up-private-endpoint-connections.md @@ -1,16 +1,16 @@ --- -title: Connect to a TiDB Dedicated Cluster via Private Endpoint with AWS +title: Connect to a TiDB Cloud Dedicated Cluster via Private Endpoint with AWS summary: Learn how to connect to your TiDB Cloud cluster via private endpoint with AWS. --- -# Connect to a TiDB Dedicated Cluster via Private Endpoint with AWS +# Connect to a TiDB Cloud Dedicated Cluster via Private Endpoint with AWS -This document describes how to connect to your TiDB Dedicated cluster via private endpoint with AWS. +This document describes how to connect to your TiDB Cloud Dedicated cluster via private endpoint with AWS. > **Tip:** > -> To learn how to connect to a TiDB Serverless cluster via private endpoint, see [Connect to TiDB Serverless via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md). -> To learn how to connect to a TiDB Dedicated cluster via private endpoint with Google Cloud, see [Connect to TiDB Dedicated via Private Service Connect with Google Cloud](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md). +> To learn how to connect to a TiDB Cloud Serverless cluster via private endpoint, see [Connect to TiDB Cloud Serverless via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md). +> To learn how to connect to a TiDB Cloud Dedicated cluster via private endpoint with Google Cloud, see [Connect to TiDB Cloud Dedicated via Private Service Connect with Google Cloud](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md). TiDB Cloud supports highly secure and one-way access to the TiDB Cloud service hosted in an AWS VPC via the [AWS PrivateLink](https://aws.amazon.com/privatelink/?privatelink-blogs.sort-by=item.additionalFields.createdDate&privatelink-blogs.sort-order=desc), as if the service were in your own VPC. A private endpoint is exposed in your VPC and you can create a connection to the TiDB Cloud service via the endpoint with permission. @@ -36,59 +36,51 @@ In most scenarios, you are recommended to use private endpoint connection over V - You are using a TiCDC cluster to replicate data to a downstream cluster (such as Amazon Aurora, MySQL, and Kafka) but you cannot maintain the endpoint service on your own. - You are connecting to PD or TiKV nodes directly. -## Set up a private endpoint with AWS +## Set up a private endpoint connection and connect to your cluster -To connect to your TiDB Dedicated cluster via a private endpoint, complete the [prerequisites](#prerequisites) and follow these steps: +To connect to your TiDB Cloud Dedicated cluster via a private endpoint, complete the follow these steps: -1. [Choose a TiDB cluster](#step-1-choose-a-tidb-cluster) -2. [Check the service endpoint region](#step-2-check-the-service-endpoint-region) -3. [Create an AWS interface endpoint](#step-3-create-an-aws-interface-endpoint) -4. [Accept the endpoint connection](#step-4-accept-the-endpoint-connection) -5. [Enable private DNS](#step-5-enable-private-dns) -6. [Connect to your TiDB cluster](#step-6-connect-to-your-tidb-cluster) +1. [Select a TiDB cluster](#step-1-select-a-tidb-cluster) +2. [Create an AWS interface endpoint](#step-2-create-an-aws-interface-endpoint) +3. [Fill in your endpoint ID](#step-3-fill-in-your-endpoint-id) +4. [Enable private DNS and create connection](#step-4-enable-private-dns-and-create-connection) +5. [Connect to your TiDB cluster](#step-5-connect-to-your-tidb-cluster) If you have multiple clusters, you need to repeat these steps for each cluster that you want to connect to using AWS PrivateLink. -### Prerequisites +### Step 1. Select a TiDB cluster -1. Log in to the [TiDB Cloud console](https://tidbcloud.com). -2. Click in the lower-left corner, switch to the target project if you have multiple projects, and then click **Project Settings**. -3. On the **Project Settings** page of your project, click **Network Access** in the left navigation pane, and click the **Private Endpoint** tab. -4. Click **Create Private Endpoint** in the upper-right corner, and then select **AWS Private Endpoint**. - -### Step 1. Choose a TiDB cluster - -1. Click the drop-down list and choose an available TiDB Dedicated cluster. -2. Click **Next**. - -### Step 2. Check the service endpoint region - -Your service endpoint region is selected by default. Have a quick check and click **Next**. +1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click the name of your target TiDB cluster to go to its overview page. +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. +3. In the **Connection Type** drop-down list, select **Private Endpoint**, and then click **Create Private Endpoint Connection**. > **Note:** > -> The default region is where your cluster is located. Do not change it. Cross-region private endpoint is currently not supported. +> If you have already created a private endpoint connection, the active endpoint will appear in the connection dialog. To create additional private endpoint connections, navigate to the **Networking** page in the left navigation pane. -### Step 3. Create an AWS interface endpoint +### Step 2. Create an AWS interface endpoint > **Note:** > -> For each TiDB Dedicated cluster created after March 28, 2023, the corresponding endpoint service is automatically created 3 to 4 minutes after the cluster creation. +> For each TiDB Cloud Dedicated cluster created after March 28, 2023, the corresponding endpoint service is automatically created 3 to 4 minutes after the cluster creation. -If you see the `Endpoint Service Ready` message, take note of your endpoint service name from the command in the lower area of the console for later use. Otherwise, wait 3 to 4 minutes to let TiDB Cloud create an endpoint service for your cluster. +If you see the `TiDB Private Link Service is ready` message, the corresponding endpoint service is ready. You can provide the following information to create the endpoint. -```bash -aws ec2 create-vpc-endpoint --vpc-id ${your_vpc_id} --region ${your_region} --service-name ${your_endpoint_service_name} --vpc-endpoint-type Interface --subnet-ids ${your_application_subnet_ids} -``` +1. Fill in the **Your VPC ID** and **Your Subnet IDs** fields. You can find these IDs from your [AWS Management Console](https://console.aws.amazon.com/). For multiple subnets, enter the IDs separated by spaces. +2. Click **Generate Command** to get the following endpoint creation command. + + ```bash + aws ec2 create-vpc-endpoint --vpc-id ${your_vpc_id} --region ${your_region} --service-name ${your_endpoint_service_name} --vpc-endpoint-type Interface --subnet-ids ${your_application_subnet_ids} + ``` -Then create an AWS interface endpoint either using the AWS Management Console or using the AWS CLI. +Then, you can create an AWS interface endpoint either using the [AWS Management Console](https://aws.amazon.com/console/) or using the AWS CLI.
    To use the AWS Management Console to create a VPC interface endpoint, perform the following steps: -1. Sign in to the [AWS Management Console](https://aws.amazon.com/console/) and open the Amazon VPC console at . +1. Sign in to the [AWS Management Console](https://aws.amazon.com/console/) and open the Amazon VPC console at [https://console.aws.amazon.com/vpc/](https://console.aws.amazon.com/vpc/). 2. Click **Endpoints** in the navigation pane, and then click **Create Endpoint** in the upper-right corner. The **Create endpoint** page is displayed. @@ -96,7 +88,7 @@ To use the AWS Management Console to create a VPC interface endpoint, perform th ![Verify endpoint service](/media/tidb-cloud/private-endpoint/create-endpoint-2.png) 3. Select **Other endpoint services**. -4. Enter the service name that you found in the TiDB Cloud console. +4. Enter the service name `${your_endpoint_service_name}` from the generated command (`--service-name ${your_endpoint_service_name}`). 5. Click **Verify service**. 6. Select your VPC in the drop-down list. 7. Select the availability zones where your TiDB cluster is located in the **Subnets** area. @@ -109,7 +101,7 @@ To use the AWS Management Console to create a VPC interface endpoint, perform th > **Note:** > - > Make sure the selected security group allows inbound access from your EC2 instances on Port 4000 or a customer-defined port. + > Make sure the selected security group allows inbound access from your EC2 instances on Port 4000 or a customer-defined port. 9. Click **Create endpoint**. @@ -118,27 +110,24 @@ To use the AWS Management Console to create a VPC interface endpoint, perform th To use the AWS CLI to create a VPC interface endpoint, perform the following steps: -1. Fill in the **VPC ID** and **Subnet IDs** fields on the private endpoint creation page. You can get the IDs from your AWS Management Console. -2. Copy the command in the lower area of the page and run it in your terminal. Then click **Next**. +1. Copy the generated command and run it in your terminal. +2. Record the VPC endpoint ID you just created. > **Tip:** > > - Before running the command, you need to have AWS CLI installed and configured. See [AWS CLI configuration basics](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html) for details. > > - If your service is spanning across more than three availability zones (AZs), you will get an error message indicating that the VPC endpoint service does not support the AZ of the subnet. This issue occurs when there is an extra AZ in your selected region in addition to the AZs where your TiDB cluster is located. In this case, you can contact [PingCAP Technical Support](https://docs.pingcap.com/tidbcloud/tidb-cloud-support). -> -> - You cannot copy the command until TiDB Cloud finishes creating an endpoint service in the background.
    -### Step 4. Accept the endpoint connection +### Step 3. Fill in your endpoint ID 1. Go back to the TiDB Cloud console. -2. Fill in the box with your VPC endpoint ID on the **Create Private Endpoint** page. -3. Click **Next**. +2. On the **Create AWS Private Endpoint Connection** page, enter your VPC endpoint ID. -### Step 5. Enable private DNS +### Step 4. Enable private DNS and create connection Enable private DNS in AWS. You can either use the AWS Management Console or the AWS CLI. @@ -157,27 +146,35 @@ To enable private DNS in your AWS Management Console:
    -To enable private DNS using your AWS CLI, copy the command and run it in your AWS CLI. +To enable private DNS using your AWS CLI, copy the following `aws ec2 modify-vpc-endpoint` command from the **Create Private Endpoint Connection** page and run it in your AWS CLI. ```bash aws ec2 modify-vpc-endpoint --vpc-endpoint-id ${your_vpc_endpoint_id} --private-dns-enabled ``` +Alternatively, you can find the command on the **Networking** page of your cluster. Locate the private endpoint and click **...*** > **Enable DNS** in the **Action** column. +
    -Click **Create** in the TiDB Cloud console to finalize the creation of the private endpoint. +Click **Create Private Endpoint Connection** in the TiDB Cloud console to finalize the creation of the private endpoint. -Then you can connect to the endpoint service. +Then you can connect to your TiDB cluster. -### Step 6. Connect to your TiDB cluster +> **Tip:** +> +> You can view and manage private endpoint connections on two pages: +> +> - Cluster-level **Networking** page: click **Networking** in the left navigation pane of the cluster overview page. +> - Project-level **Network Access** page: click **Network Access** in the left navigation pane of the **Project Settings** page. + +### Step 5. Connect to your TiDB cluster -After you have enabled the private DNS, go back to the TiDB Cloud console and take the following steps: +After you have accepted the private endpoint connection, you are redirected back to the connection dialog. -1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click **...** in the **Action** column. -2. Click **Connect**. A connection dialog is displayed. -3. Select the **Private Endpoint** tab. The private endpoint you just created is displayed under **Step 1: Create Private Endpoint**. -4. Under **Step 2: Connect your connection**, click **Connect**, click the tab of your preferred connection method, and then connect to your cluster with the connection string. The placeholders `:` in the connection string are automatically replaced with the real values. +1. Wait for the private endpoint connection status to change from **System Checking** to **Active** (approximately 5 minutes). +2. In the **Connect With** drop-down list, select your preferred connection method. The corresponding connection string is displayed at the bottom of the dialog. +3. Connect to your cluster with the connection string. > **Tip:** > @@ -185,7 +182,10 @@ After you have enabled the private DNS, go back to the TiDB Cloud console and ta ### Private endpoint status reference -When you use private endpoint connections, the statuses of private endpoints or private endpoint services are displayed on the [**Private Endpoint** page](#prerequisites). +When you use private endpoint connections, the statuses of private endpoints or private endpoint services are displayed on the following pages: + +- Cluster-level **Networking** page: click **Networking** in the left navigation pane of the cluster overview page. +- Project-level **Network Access** page: click **Network Access** in the left navigation pane of the **Project Settings** page. The possible statuses of a private endpoint are explained as follows: diff --git a/tidb-cloud/set-up-vpc-peering-connections.md b/tidb-cloud/set-up-vpc-peering-connections.md index 23f6986e2b777..2bdc751469c46 100644 --- a/tidb-cloud/set-up-vpc-peering-connections.md +++ b/tidb-cloud/set-up-vpc-peering-connections.md @@ -1,53 +1,55 @@ --- -title: Connect to TiDB Dedicated via VPC Peering -summary: Learn how to connect to TiDB Dedicated via VPC peering. +title: Connect to TiDB Cloud Dedicated via VPC Peering +summary: Learn how to connect to TiDB Cloud Dedicated via VPC peering. --- -# Connect to TiDB Dedicated via VPC Peering +# Connect to TiDB Cloud Dedicated via VPC Peering > **Note:** > -> VPC peering connection is only available for TiDB Dedicated clusters. You cannot connect to [TiDB Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-serverless) using VPC peering. +> VPC peering connection is only available for TiDB Cloud Dedicated clusters. You cannot connect to [TiDB Cloud Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) using VPC peering. To connect your application to TiDB Cloud via VPC peering, you need to set up [VPC peering](/tidb-cloud/tidb-cloud-glossary.md#vpc-peering) with TiDB Cloud. This document walks you through setting up VPC peering connections [on AWS](#set-up-vpc-peering-on-aws) and [on Google Cloud](#set-up-vpc-peering-on-google-cloud) and connecting to TiDB Cloud via a VPC peering. VPC peering connection is a networking connection between two VPCs that enables you to route traffic between them using private IP addresses. Instances in either VPC can communicate with each other as if they are within the same network. -Currently, TiDB Cloud only supports VPC peering in the same region for the same project. TiDB clusters of the same project in the same region are created in the same VPC. Therefore, once VPC peering is set up in a region of a project, all the TiDB clusters created in the same region of this project can be connected in your VPC. VPC peering setup differs among cloud providers. +Currently, TiDB clusters of the same project in the same region are created in the same VPC. Therefore, once VPC peering is set up in a region of a project, all the TiDB clusters created in the same region of this project can be connected in your VPC. VPC peering setup differs among cloud providers. > **Tip:** > > To connect your application to TiDB Cloud, you can also set up [private endpoint connection](/tidb-cloud/set-up-private-endpoint-connections.md) with TiDB Cloud, which is secure and private, and does not expose your data to the public internet. It is recommended to use private endpoints over VPC peering connections. -## Prerequisite: Set a Project CIDR +## Prerequisite: Set a CIDR for a region -Project CIDR (Classless Inter-Domain Routing) is the CIDR block used for network peering in a project. +CIDR (Classless Inter-Domain Routing) is the CIDR block used for creating VPC for TiDB Cloud Dedicated clusters. -Before adding VPC Peering requests to a region, you need to set a project CIDR for your project's cloud provider (AWS or Google Cloud) to establish a peering link to your application's VPC. +Before adding VPC Peering requests to a region, you must set a CIDR for that region and create an initial TiDB Cloud Dedicated cluster in that region. Once the first Dedicated cluster is created, TiDB Cloud will create the VPC of the cluster, allowing you to establish a peering link to your application's VPC. -You can set the project CIDR when creating the first TiDB Dedicated of your project. If you want to set the project CIDR before creating the cluster, perform the following operations: +You can set the CIDR when creating the first TiDB Cloud Dedicated cluster. If you want to set the CIDR before creating the cluster, perform the following operations: 1. Log in to the [TiDB Cloud console](https://tidbcloud.com). 2. Click in the lower-left corner, switch to the target project if you have multiple projects, and then click **Project Settings**. -3. On the **Project Settings** page of your project, click **Network Access** in the left navigation pane, and then click the **Project CIDR** tab. -4. Click **Add a project CIDR for AWS** or **Add a project CIDR for Google Cloud** according to your cloud provider, specify one of the following network addresses in the **Project CIDR** field, and then click **Confirm**. +3. On the **Project Settings** page of your project, click **Network Access** in the left navigation pane, click the **Project CIDR** tab, and then select **AWS** or **Google Cloud** according to your cloud provider. +4. In the upper-right corner, click **Create CIDR**. Specify the region and CIDR value in the **Create AWS CIDR** or **Create Google Cloud CIDR** dialog, and then click **Confirm**. + + ![Project-CIDR4](/media/tidb-cloud/Project-CIDR4.png) > **Note:** > - > To avoid any conflicts with the CIDR of the VPC where your application is located, you need to set a different project CIDR in this field. - - - 10.250.0.0/16 - - 10.250.0.0/17 - - 10.250.128.0/17 - - 172.30.0.0/16 - - 172.30.0.0/17 - - 172.30.128.0/17 - - ![Project-CIDR4](/media/tidb-cloud/Project-CIDR4.png) + > - To avoid any conflicts with the CIDR of the VPC where your application is located, you need to set a different project CIDR in this field. + > - For AWS Region, it is recommended to configure an IP range size between `/16` and `/23`. Supported network addresses include: + > - 10.250.0.0 - 10.251.255.255 + > - 172.16.0.0 - 172.31.255.255 + > - 192.168.0.0 - 192.168.255.255 + > - For Google Cloud Region, it is recommended to configure an IP range size between `/19` and `/20`. If you want to configure an IP range size between `/16` and `/18`, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). Supported network addresses include: + > - 10.250.0.0 - 10.251.255.255 + > - 172.16.0.0 - 172.17.255.255 + > - 172.30.0.0 - 172.31.255.255 + > - TiDB Cloud limits the number of TiDB Cloud nodes in a region of a project based on the CIDR block size of the region. 5. View the CIDR of the cloud provider and the specific region. - The region CIDR is inactive by default. To activate the region CIDR, you need to create a cluster in the target region. When the region CIDR is active, you can create VPC Peering for the region. + The CIDR is inactive by default. To activate the CIDR, you need to create a cluster in the target region. When the region CIDR is active, you can create VPC Peering for the region. ![Project-CIDR2](/media/tidb-cloud/Project-CIDR2.png) @@ -57,33 +59,71 @@ This section describes how to set up VPC peering connections on AWS. For Google ### Step 1. Add VPC peering requests +You can add VPC peering requests on either the project-level **Network Access** page or the cluster-level **Networking** page in the TiDB Cloud console. + + +
    + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com). 2. Click in the lower-left corner, switch to the target project if you have multiple projects, and then click **Project Settings**. -3. On the **Project Settings** page of your project, click **Network Access** in the left navigation pane, and then click the **VPC Peering** tab. +3. On the **Project Settings** page of your project, click **Network Access** in the left navigation pane, and click the **VPC Peering** > **AWS** tab. The **VPC Peering** configuration is displayed by default. -4. Click **Add**, choose the AWS icon, and then fill in the required information of your existing AWS VPC: +4. In the upper-right corner, click **Create VPC Peering**, select the **TiDB Cloud VPC Region**, and then fill in the required information of your existing AWS VPC: - - Region + - Your VPC Region - AWS Account ID - VPC ID - VPC CIDR - You can get these information from your VPC details on the VPC dashboard. + You can get such information from your VPC details page of the [AWS Management Console](https://console.aws.amazon.com/). TiDB Cloud supports creating VPC peerings between VPCs in the same region or from two different regions. ![VPC peering](/media/tidb-cloud/vpc-peering/vpc-peering-creating-infos.png) -5. Click **Initialize**. The **Approve VPC Peerings** dialog is displayed. +5. Click **Create** to send the VPC peering request, and then view the VPC peering information on the **VPC Peering** > **AWS** tab. The status of the newly created VPC peering is **System Checking**. -### Step 2. Approve and configure the VPC peering +6. To view detailed information about your newly created VPC peering, click **...** > **View** in the **Action** column. The **VPC Peering Details** page is displayed. + +
    +
    + +1. Open the overview page of the target cluster. + + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + + 2. Click the name of your target cluster to go to its overview page. -Use either of the following two options to approve and configure the VPC peering connection: +2. In the left navigation pane, click **Networking**, and then click **Create VPC Peering**. + +3. Fill in the required information of your existing AWS VPC: + + - Your VPC Region + - AWS Account ID + - VPC ID + - VPC CIDR + + You can get such information from your VPC details page of the [AWS Management Console](https://console.aws.amazon.com/). TiDB Cloud supports creating VPC peerings between VPCs in the same region or from two different regions. + + ![VPC peering](/media/tidb-cloud/vpc-peering/vpc-peering-creating-infos.png) + +4. Click **Create** to send the VPC peering request, and then view the VPC peering information on the **Networking** > **AWS VPC Peering** section. The status of the newly created VPC peering is **System Checking**. + +5. To view detailed information about your newly created VPC peering, click **...** > **View** in the **Action** column. The **AWS VPC Peering Details** page is displayed. + +
    +
    + +### Step 2. Approve and configure the VPC peering -- [Option 1: Use AWS CLI](#option-1-use-aws-cli) -- [Option 2: Use the AWS dashboard](#option-2-use-the-aws-dashboard) +You can approve and configure the VPC peering connection using AWS CLI or AWS dashboard. -#### Option 1. Use AWS CLI + +
    1. Install AWS Command Line Interface (AWS CLI). @@ -141,7 +181,7 @@ Use either of the following two options to approve and configure the VPC peering aws ec2 describe-route-tables --region "$app_region" --filters Name=vpc-id,Values="$app_vpc_id" --query 'RouteTables[*].RouteTableId' --output text | tr "\t" "\n" | while read row do app_route_table_id="$row" - aws ec2 create-route --route-table-id "$app_route_table_id" --destination-cidr-block "$tidbcloud_project_cidr" --vpc-peering-connection-id "$pcx_tidb_to_app_id" + aws ec2 create-route --region "$app_region" --route-table-id "$app_route_table_id" --destination-cidr-block "$tidbcloud_project_cidr" --vpc-peering-connection-id "$pcx_tidb_to_app_id" done ``` @@ -159,19 +199,20 @@ Use either of the following two options to approve and configure the VPC peering After finishing the configuration, the VPC peering has been created. You can [connect to the TiDB cluster](#connect-to-the-tidb-cluster) to verify the result. -#### Option 2. Use the AWS dashboard +
    +
    You can also use the AWS dashboard to configure the VPC peering connection. -1. Confirm to accept the peer connection request in your AWS console. +1. Confirm to accept the peer connection request in your [AWS Management Console](https://console.aws.amazon.com/). - 1. Sign in to the AWS console and click **Services** on the top menu bar. Enter `VPC` in the search box and go to the VPC service page. + 1. Sign in to the [AWS Management Console](https://console.aws.amazon.com/) and click **Services** on the top menu bar. Enter `VPC` in the search box and go to the VPC service page. ![AWS dashboard](/media/tidb-cloud/vpc-peering/aws-vpc-guide-1.jpg) 2. From the left navigation bar, open the **Peering Connections** page. On the **Create Peering Connection** tab, a peering connection is in the **Pending Acceptance** status. - 3. Confirm the requester owner is TiDB Cloud (`380838443567`). Right-click the peering connection and select **Accept Request** to accept the request in the **Accept VPC peering connection request** dialog. + 3. Confirm that the requester owner and the requester VPC match **TiDB Cloud AWS Account ID** and **TiDB Cloud VPC ID** on the **VPC Peering Details** page of the [TiDB Cloud console](https://tidbcloud.com). Right-click the peering connection and select **Accept Request** to accept the request in the **Accept VPC peering connection request** dialog. ![AWS VPC peering requests](/media/tidb-cloud/vpc-peering/aws-vpc-guide-3.png) @@ -183,7 +224,7 @@ You can also use the AWS dashboard to configure the VPC peering connection. ![Search all route tables related to VPC](/media/tidb-cloud/vpc-peering/aws-vpc-guide-4.png) - 3. Right-click each route table and select **Edit routes**. On the edit page, add a route with a destination to the Project CIDR (by checking the **VPC Peering** configuration page in the TiDB Cloud console) and fill in your peering connection ID in the **Target** column. + 3. Right-click each route table and select **Edit routes**. On the edit page, add a route with a destination to the TiDB Cloud CIDR (by checking the **VPC Peering** configuration page in the TiDB Cloud console) and fill in your peering connection ID in the **Target** column. ![Edit all route tables](/media/tidb-cloud/vpc-peering/aws-vpc-guide-5.png) @@ -201,42 +242,81 @@ You can also use the AWS dashboard to configure the VPC peering connection. Now you have successfully set up the VPC peering connection. Next, [connect to the TiDB cluster via VPC peering](#connect-to-the-tidb-cluster). +
    +
    + ## Set up VPC peering on Google Cloud +### Step 1. Add VPC peering requests + +You can add VPC peering requests on either the project-level **Network Access** page or the cluster-level **Networking** page in the TiDB Cloud console. + + +
    + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com). 2. Click in the lower-left corner, switch to the target project if you have multiple projects, and then click **Project Settings**. -3. On the **Project Settings** page of your project, click **Network Access** in the left navigation pane, and then click the **VPC Peering** tab. +3. On the **Project Settings** page of your project, click **Network Access** in the left navigation pane, and click the **VPC Peering** > **Google Cloud** tab. The **VPC Peering** configuration is displayed by default. -4. Click **Add**, choose the Google Cloud icon, and then fill in the required information of your existing Google Cloud VPC: +4. In the upper-right corner, click **Create VPC Peering**, select the **TiDB Cloud VPC Region**, and then fill in the required information of your existing Google Cloud VPC: > **Tip:** > - > You can follow instructions next to the **Application Google Cloud Project ID** and **VPC Network Name** fields to find the project ID and VPC network name. + > You can follow instructions next to the **Google Cloud Project ID** and **VPC Network Name** fields to find the project ID and VPC network name. - - Region - - Application Google Cloud Project ID + - Google Cloud Project ID - VPC Network Name - VPC CIDR -5. Click **Initialize**. The **Approve VPC Peerings** dialog is displayed. +5. Click **Create** to send the VPC peering request, and then view the VPC peering information on the **VPC Peering** > **Google Cloud** tab. The status of the newly created VPC peering is **System Checking**. -6. Check the connection information of your TiDB VPC peerings. +6. To view detailed information about your newly created VPC peering, click **...** > **View** in the **Action** column. The **VPC Peering Details** page is displayed. - ![VPC-Peering](/media/tidb-cloud/VPC-Peering3.png) +
    +
    -7. Execute the following command to finish the setup of VPC peerings: +1. Open the overview page of the target cluster. - {{< copyable "shell-regular" >}} + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. - ```bash - gcloud beta compute networks peerings create --project --network --peer-project --peer-network - ``` + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. - > **Note:** + 2. Click the name of your target cluster to go to its overview page. + +2. In the left navigation pane, click **Networking**, and then click **Create VPC Peering**. + +3. Fill in the required information of your existing Google Cloud VPC: + + > **Tip:** > - > You can name `` as you like. + > You can follow instructions next to the **Google Cloud Project ID** and **VPC Network Name** fields to find the project ID and VPC network name. + + - Google Cloud Project ID + - VPC Network Name + - VPC CIDR + +4. Click **Create** to send the VPC peering request, and then view the VPC peering information on the **Networking** > **Google Cloud VPC Peering** section. The status of the newly created VPC peering is **System Checking**. + +5. To view detailed information about your newly created VPC peering, click **...** > **View** in the **Action** column. The **Google Cloud VPC Peering Details** page is displayed. + +
    +
    + +### Step 2. Approve the VPC peering + +Execute the following command to finish the setup of VPC peering: + +```bash +gcloud beta compute networks peerings create --project --network --peer-project --peer-network +``` + +> **Note:** +> +> You can name `` as you like. Now you have successfully set up the VPC peering connection. Next, [connect to the TiDB cluster via VPC peering](#connect-to-the-tidb-cluster). @@ -244,10 +324,10 @@ Now you have successfully set up the VPC peering connection. Next, [connect to t 1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click the name of your target cluster to go to its overview page. -2. Click **Connect** in the upper-right corner, and select the **VPC Peering** tab in the connection dialog. +2. Click **Connect** in the upper-right corner, and select **VPC Peering** from the **Connection Type** drop-down list. - You can see the **Status** of the VPC peering is **active**. If **Status** is still **system checking**, wait for about 5 minutes and open the dialog again. + Wait for the VPC peering connection status to change from **system checking** to **active** (approximately 5 minutes). -3. Click **Get Endpoint** and wait for a few minutes. Then the connection command is displayed in the dialog. +3. In the **Connect With** drop-down list, select your preferred connection method. The corresponding connection string is displayed at the bottom of the dialog. -4. Under **Step 2: Connect with a SQL client** in the dialog box, click the tab of your preferred connection method, and then connect to your cluster with the connection string. +4. Connect to your cluster with the connection string. diff --git a/tidb-cloud/setup-self-hosted-kafka-private-link-service.md b/tidb-cloud/setup-self-hosted-kafka-private-link-service.md new file mode 100644 index 0000000000000..f6170830d41f6 --- /dev/null +++ b/tidb-cloud/setup-self-hosted-kafka-private-link-service.md @@ -0,0 +1,761 @@ +--- +title: Set Up Self-Hosted Kafka Private Link Service in AWS +summary: This document explains how to set up Private Link service for self-hosted Kafka in AWS and how to make it work with TiDB Cloud. +--- + +# Set Up Self-Hosted Kafka Private Link Service in AWS + +This document describes how to set up Private Link service for self-hosted Kafka in AWS, and how to make it work with TiDB Cloud. + +The mechanism works as follows: + +1. The TiDB Cloud VPC connects to the Kafka VPC through private endpoints. +2. Kafka clients need to communicate directly to all Kafka brokers. +3. Each Kafka broker is mapped to a unique port of endpoints within the TiDB Cloud VPC. +4. Leverage the Kafka bootstrap mechanism and AWS resources to achieve the mapping. + +The following diagram shows the mechanism. + +![Connect to AWS Self-Hosted Kafka Private Link Service](/media/tidb-cloud/changefeed/connect-to-aws-self-hosted-kafka-privatelink-service.jpeg) + +The document provides an example of connecting to a Kafka Private Link service deployed across three availability zones (AZ) in AWS. While other configurations are possible based on similar port-mapping principles, this document covers the fundamental setup process of the Kafka Private Link service. For production environments, a more resilient Kafka Private Link service with enhanced operational maintainability and observability is recommended. + +## Prerequisites + +1. Ensure that you have the following authorization to set up a Kafka Private Link service in your own AWS account. + + - Manage EC2 nodes + - Manage VPC + - Manage subnets + - Manage security groups + - Manage load balancer + - Manage endpoint services + - Connect to EC2 nodes to configure Kafka nodes + +2. [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md) if you do not have one. + +3. Get the Kafka deployment information from your TiDB Cloud Dedicated cluster. + + 1. In the [TiDB Cloud console](https://tidbcloud.com), navigate to the cluster overview page of the TiDB cluster, and then click **Changefeed** in the left navigation pane. + 2. On the overview page, find the region of the TiDB cluster. Ensure that your Kafka cluster will be deployed to the same region. + 3. Click **Create Changefeed**. + 1. In **Target Type**, select **Kafka**. + 2. In **Connectivity Method**, select **Private Link**. + 4. Note down the information of the TiDB Cloud AWS account in **Reminders before proceeding**. You will use it to authorize TiDB Cloud to create an endpoint for the Kafka Private Link service. + 5. Select **Number of AZs**. In this example, select **3 AZs**. Note down the IDs of the AZs in which you want to deploy your Kafka cluster. If you want to know the relationship between your AZ names and AZ IDs, see [Availability Zone IDs for your AWS resources](https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html) to find it. + 6. Enter a unique **Kafka Advertised Listener Pattern** for your Kafka Private Link service. + 1. Input a unique random string. It can only include numbers or lowercase letters. You will use it to generate **Kafka Advertised Listener Pattern** later. + 2. Click **Check usage and generate** to check if the random string is unique and generate **Kafka Advertised Listener Pattern** that will be used to assemble the EXTERNAL advertised listener for Kafka brokers. + +Note down all the deployment information. You need to use it to configure your Kafka Private Link service later. + +The following table shows an example of the deployment information. + +| Information | Value | Note | +|--------|-----------------|---------------------------| +| Region | Oregon (`us-west-2`) | N/A | +| Principal of TiDB Cloud AWS Account | `arn:aws:iam:::root` | N/A | +| AZ IDs |
    • `usw2-az1`
    • `usw2-az2`
    • `usw2-az3`
    | Align AZ IDs to AZ names in your AWS account.
    Example:
    • `usw2-az1` => `us-west-2a`
    • `usw2-az2` => `us-west-2c`
    • `usw2-az3` => `us-west-2b`
    | +| Kafka Advertised Listener Pattern | The unique random string: `abc`
    Generated pattern for AZs:
    • `usw2-az1` => <broker_id>.usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:<port>
    • `usw2-az2` => <broker_id>.usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:<port>
    • `usw2-az3` => <broker_id>.usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:<port>
    | Map AZ names to AZ-specified patterns. Make sure that you configure the right pattern to the broker in a specific AZ later.
    • `us-west-2a` => <broker_id>.usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:<port>
    • `us-west-2c` => <broker_id>.usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:<port>
    • `us-west-2b` => <broker_id>.usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:<port>
    | + +## Step 1. Set up a Kafka cluster + +If you need to deploy a new cluster, follow the instructions in [Deploy a new Kafka cluster](#deploy-a-new-kafka-cluster). + +If you need to expose an existing cluster, follow the instructions in [Reconfigure a running Kafka cluster](#reconfigure-a-running-kafka-cluster). + +### Deploy a new Kafka cluster + +#### 1. Set up the Kafka VPC + +The Kafka VPC requires the following: + +- Three private subnets for brokers, one for each AZ. +- One public subnet in any AZ with a bastion node that can connect to the internet and three private subnets, which makes it easy to set up the Kafka cluster. In a production environment, you might have your own bastion node that can connect to the Kafka VPC. + +Before creating subnets, create subnets in AZs based on the mappings of AZ IDs and AZ names. Take the following mapping as an example. + +- `usw2-az1` => `us-west-2a` +- `usw2-az2` => `us-west-2c` +- `usw2-az3` => `us-west-2b` + +Create private subnets in the following AZs: + +- `us-west-2a` +- `us-west-2c` +- `us-west-2b` + +Take the following steps to create the Kafka VPC. + +**1.1. Create the Kafka VPC** + +1. Go to [AWS Console > VPC dashboard](https://console.aws.amazon.com/vpcconsole/home?#vpcs:), and switch to the region in which you want to deploy Kafka. + +2. Click **Create VPC**. Fill in the information on the **VPC settings** page as follows. + + 1. Select **VPC only**. + 2. Enter a tag in **Name tag**, for example, `Kafka VPC`. + 3. Select **IPv4 CIDR manual input**, and enter the IPv4 CIDR, for example, `10.0.0.0/16`. + 4. Use the default values for other options. Click **Create VPC**. + 5. On the VPC detail page, take note of the VPC ID, for example, `vpc-01f50b790fa01dffa`. + +**1.2. Create private subnets in the Kafka VPC** + +1. Go to the [Subnets Listing page](https://console.aws.amazon.com/vpcconsole/home?#subnets:). +2. Click **Create subnet**. +3. Select **VPC ID** (`vpc-01f50b790fa01dffa` in this example) that you noted down before. +4. Add three subnets with the following information. It is recommended that you put the AZ IDs in the subnet names to make it easy to configure the brokers later, because TiDB Cloud requires encoding the AZ IDs in the broker's `advertised.listener` configuration. + + - Subnet1 in `us-west-2a` + - **Subnet name**: `broker-usw2-az1` + - **Availability Zone**: `us-west-2a` + - **IPv4 subnet CIDR block**: `10.0.0.0/18` + + - Subnet2 in `us-west-2c` + - **Subnet name**: `broker-usw2-az2` + - **Availability Zone**: `us-west-2c` + - **IPv4 subnet CIDR block**: `10.0.64.0/18` + + - Subnet3 in `us-west-2b` + - **Subnet name**: `broker-usw2-az3` + - **Availability Zone**: `us-west-2b` + - **IPv4 subnet CIDR block**: `10.0.128.0/18` + +5. Click **Create subnet**. The **Subnets Listing** page is displayed. + +**1.3. Create the public subnet in the Kafka VPC** + +1. Click **Create subnet**. +2. Select **VPC ID** (`vpc-01f50b790fa01dffa` in this example) that you noted down before. +3. Add the public subnet in any AZ with the following information: + + - **Subnet name**: `bastion` + - **IPv4 subnet CIDR block**: `10.0.192.0/18` + +4. Click **Create subnet**. The **Subnets Listing** page is displayed. +5. Configure the bastion subnet to the Public subnet. + + 1. Go to [VPC dashboard > Internet gateways](https://console.aws.amazon.com/vpcconsole/home#igws:). Create an Internet Gateway with the name `kafka-vpc-igw`. + 2. On the **Internet gateways Detail** page, in **Actions**, click **Attach to VPC** to attach the Internet Gateway to the Kafka VPC. + 3. Go to [VPC dashboard > Route tables](https://console.aws.amazon.com/vpcconsole/home#CreateRouteTable:). Create a route table to the Internet Gateway in Kafka VPC and add a new route with the following information: + + - **Name**: `kafka-vpc-igw-route-table` + - **VPC**: `Kafka VPC` + - **Route**: + - **Destination**: `0.0.0.0/0` + - **Target**: `Internet Gateway`, `kafka-vpc-igw` + + 4. Attach the route table to the bastion subnet. On the **Detail** page of the route table, click **Subnet associations > Edit subnet associations** to add the bastion subnet and save changes. + +#### 2. Set up Kafka brokers + +**2.1. Create a bastion node** + +Go to the [EC2 Listing page](https://console.aws.amazon.com/ec2/home#Instances:). Create the bastion node in the bastion subnet. + +- **Name**: `bastion-node` +- **Amazon Machine Image**: `Amazon linux` +- **Instance Type**: `t2.small` +- **Key pair**: `kafka-vpc-key-pair`. Create a new key pair named `kafka-vpc-key-pair`. Download **kafka-vpc-key-pair.pem** to your local for later configuration. +- Network settings + + - **VPC**: `Kafka VPC` + - **Subnet**: `bastion` + - **Auto-assign public IP**: `Enable` + - **Security Group**: create a new security group allow SSH login from anywhere. You can narrow the rule for safety in the production environment. + +**2.2. Create broker nodes** + +Go to the [EC2 Listing page](https://console.aws.amazon.com/ec2/home#Instances:). Create three broker nodes in broker subnets, one for each AZ. + +- Broker 1 in subnet `broker-usw2-az1` + + - **Name**: `broker-node1` + - **Amazon Machine Image**: `Amazon linux` + - **Instance Type**: `t2.large` + - **Key pair**: reuse `kafka-vpc-key-pair` + - Network settings + + - **VPC**: `Kafka VPC` + - **Subnet**: `broker-usw2-az1` + - **Auto-assign public IP**: `Disable` + - **Security Group**: create a new security group to allow all TCP from Kafka VPC. You can narrow the rule for safety in the production environment. + - **Protocol**: `TCP` + - **Port range**: `0 - 65535` + - **Source**: `10.0.0.0/16` + +- Broker 2 in subnet `broker-usw2-az2` + + - **Name**: `broker-node2` + - **Amazon Machine Image**: `Amazon linux` + - **Instance Type**: `t2.large` + - **Key pair**: reuse `kafka-vpc-key-pair` + - Network settings + + - **VPC**: `Kafka VPC` + - **Subnet**: `broker-usw2-az2` + - **Auto-assign public IP**: `Disable` + - **Security Group**: create a new security group to allow all TCP from Kafka VPC. You can narrow the rule for safety in the production environment. + - **Protocol**: `TCP` + - **Port range**: `0 - 65535` + - **Source**: `10.0.0.0/16` + +- Broker 3 in subnet `broker-usw2-az3` + + - **Name**: `broker-node3` + - **Amazon Machine Image**: `Amazon linux` + - **Instance Type**: `t2.large` + - **Key pair**: reuse `kafka-vpc-key-pair` + - Network settings + + - **VPC**: `Kafka VPC` + - **Subnet**: `broker-usw2-az3` + - **Auto-assign public IP**: `Disable` + - **Security Group**: create a new security group to allow all TCP from Kafka VPC. You can narrow the rule for safety in the production environment. + - **Protocol**: `TCP` + - **Port range**: `0 - 65535` + - **Source**: `10.0.0.0/16` + +**2.3. Prepare Kafka runtime binaries** + +1. Go to the detail page of the bastion node. Get the **Public IPv4 address**. Use SSH to log in to the node with the previously downloaded `kafka-vpc-key-pair.pem`. + + ```shell + chmod 400 kafka-vpc-key-pair.pem + ssh -i "kafka-vpc-key-pair.pem" ec2-user@{bastion_public_ip} # replace {bastion_public_ip} with the IP address of your bastion node, for example, 54.186.149.187 + scp -i "kafka-vpc-key-pair.pem" kafka-vpc-key-pair.pem ec2-user@{bastion_public_ip}:~/ + ``` + +2. Download binaries. + + ```shell + # Download Kafka and OpenJDK, and then extract the files. You can choose the binary version based on your preference. + wget https://downloads.apache.org/kafka/3.7.1/kafka_2.13-3.7.1.tgz + tar -zxf kafka_2.13-3.7.1.tgz + wget https://download.java.net/java/GA/jdk22.0.2/c9ecb94cd31b495da20a27d4581645e8/9/GPL/openjdk-22.0.2_linux-x64_bin.tar.gz + tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz + ``` + +3. Copy binaries to each broker node. + + ```shell + # Replace {broker-node1-ip} with your broker-node1 IP address + scp -i "kafka-vpc-key-pair.pem" kafka_2.13-3.7.1.tgz ec2-user@{broker-node1-ip}:~/ + ssh -i "kafka-vpc-key-pair.pem" ec2-user@{broker-node1-ip} "tar -zxf kafka_2.13-3.7.1.tgz" + scp -i "kafka-vpc-key-pair.pem" openjdk-22.0.2_linux-x64_bin.tar.gz ec2-user@{broker-node1-ip}:~/ + ssh -i "kafka-vpc-key-pair.pem" ec2-user@{broker-node1-ip} "tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz" + + # Replace {broker-node2-ip} with your broker-node2 IP address + scp -i "kafka-vpc-key-pair.pem" kafka_2.13-3.7.1.tgz ec2-user@{broker-node2-ip}:~/ + ssh -i "kafka-vpc-key-pair.pem" ec2-user@{broker-node2-ip} "tar -zxf kafka_2.13-3.7.1.tgz" + scp -i "kafka-vpc-key-pair.pem" openjdk-22.0.2_linux-x64_bin.tar.gz ec2-user@{broker-node2-ip}:~/ + ssh -i "kafka-vpc-key-pair.pem" ec2-user@{broker-node2-ip} "tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz" + + # Replace {broker-node3-ip} with your broker-node3 IP address + scp -i "kafka-vpc-key-pair.pem" kafka_2.13-3.7.1.tgz ec2-user@{broker-node3-ip}:~/ + ssh -i "kafka-vpc-key-pair.pem" ec2-user@{broker-node3-ip} "tar -zxf kafka_2.13-3.7.1.tgz" + scp -i "kafka-vpc-key-pair.pem" openjdk-22.0.2_linux-x64_bin.tar.gz ec2-user@{broker-node3-ip}:~/ + ssh -i "kafka-vpc-key-pair.pem" ec2-user@{broker-node3-ip} "tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz" + ``` + +**2.4. Set up Kafka nodes on each broker node** + +**2.4.1 Set up a KRaft Kafka cluster with three nodes** + +Each node will act as a broker and controller role. Do the following for each broker: + +1. For the `listeners` item, all three brokers are the same and act as broker and controller roles: + + 1. Configure the same CONTROLLER listener for all **controller** role nodes. If you only want to add the **broker** role nodes, you do not need the CONTROLLER listener in `server.properties`. + 2. Configure two **broker** listeners, `INTERNAL` for internal access and `EXTERNAL` for external access from TiDB Cloud. + +2. For the `advertised.listeners` item, do the following: + + 1. Configure an INTERNAL advertised listener for every broker with the internal IP of the broker node. Advertised internal Kafka clients use this address to visit the broker. + 2. Configure an EXTERNAL advertised listener based on **Kafka Advertised Listener Pattern** you get from TiDB Cloud for each broker node to help TiDB Cloud differentiate between different brokers. Different EXTERNAL advertised listeners help the Kafka client from TiDB Cloud route requests to the right broker. + + - `` differentiates brokers from Kafka Private Link Service access points. Plan a port range for EXTERNAL advertised listeners of all brokers. These ports do not have to be actual ports listened to by brokers. They are ports listened to by the load balancer for Private Link Service that will forward requests to different brokers. + - `AZ ID` in **Kafka Advertised Listener Pattern** indicates where the broker is deployed. TiDB Cloud will route requests to different endpoint DNS names based on the AZ ID. + + It is recommended to configure different broker IDs for different brokers to make it easy for troubleshooting. + +3. The planning values are as follows: + + - **CONTROLLER port**: `29092` + - **INTERNAL port**: `9092` + - **EXTERNAL**: `39092` + - **EXTERNAL advertised listener ports range**: `9093~9095` + +**2.4.2. Create a configuration file** + +Use SSH to log in to every broker node. Create a configuration file `~/config/server.properties` with the following content. + +```properties +# brokers in usw2-az1 + +# broker-node1 ~/config/server.properties +# 1. Replace {broker-node1-ip}, {broker-node2-ip}, {broker-node3-ip} with the actual IP addresses. +# 2. Configure EXTERNAL in "advertised.listeners" based on the "Kafka Advertised Listener Pattern" in the "Prerequisites" section. +# 2.1 The pattern for AZ(ID: usw2-az1) is ".usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:". +# 2.2 So the EXTERNAL can be "b1.usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:9093". Replace with "b" prefix plus "node.id" properties, and replace with a unique port (9093) in the port range of the EXTERNAL advertised listener. +# 2.3 If there are more broker role nodes in the same AZ, you can configure them in the same way. +process.roles=broker,controller +node.id=1 +controller.quorum.voters=1@{broker-node1-ip}:29092,2@{broker-node2-ip}:29092,3@{broker-node3-ip}:29092 +listeners=INTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29092,EXTERNAL://0.0.0.0:39092 +inter.broker.listener.name=INTERNAL +advertised.listeners=INTERNAL://{broker-node1-ip}:9092,EXTERNAL://b1.usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:9093 +controller.listener.names=CONTROLLER +listener.security.protocol.map=INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL +log.dirs=./data +``` + +```properties +# brokers in usw2-az2 + +# broker-node2 ~/config/server.properties +# 1. Replace {broker-node1-ip}, {broker-node2-ip}, {broker-node3-ip} with the actual IP addresses. +# 2. Configure EXTERNAL in "advertised.listeners" based on the "Kafka Advertised Listener Pattern" in the "Prerequisites" section. +# 2.1 The pattern for AZ(ID: usw2-az2) is ".usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:". +# 2.2 So the EXTERNAL can be "b2.usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:9094". Replace with "b" prefix plus "node.id" properties, and replace with a unique port (9094) in the port range of the EXTERNAL advertised listener. +# 2.3 If there are more broker role nodes in the same AZ, you can configure them in the same way. +process.roles=broker,controller +node.id=2 +controller.quorum.voters=1@{broker-node1-ip}:29092,2@{broker-node2-ip}:29092,3@{broker-node3-ip}:29092 +listeners=INTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29092,EXTERNAL://0.0.0.0:39092 +inter.broker.listener.name=INTERNAL +advertised.listeners=INTERNAL://{broker-node2-ip}:9092,EXTERNAL://b2.usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:9094 +controller.listener.names=CONTROLLER +listener.security.protocol.map=INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL +log.dirs=./data +``` + +```properties +# brokers in usw2-az3 + +# broker-node3 ~/config/server.properties +# 1. Replace {broker-node1-ip}, {broker-node2-ip}, {broker-node3-ip} with the actual IP addresses. +# 2. Configure EXTERNAL in "advertised.listeners" based on the "Kafka Advertised Listener Pattern" in the "Prerequisites" section. +# 2.1 The pattern for AZ(ID: usw2-az3) is ".usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:". +# 2.2 So the EXTERNAL can be "b3.usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:9095". Replace with "b" prefix plus "node.id" properties, and replace with a unique port (9095) in the port range of the EXTERNAL advertised listener. +# 2.3 If there are more broker role nodes in the same AZ, you can configure them in the same way. +process.roles=broker,controller +node.id=3 +controller.quorum.voters=1@{broker-node1-ip}:29092,2@{broker-node2-ip}:29092,3@{broker-node3-ip}:29092 +listeners=INTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29092,EXTERNAL://0.0.0.0:39092 +inter.broker.listener.name=INTERNAL +advertised.listeners=INTERNAL://{broker-node3-ip}:9092,EXTERNAL://b3.usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:9095 +controller.listener.names=CONTROLLER +listener.security.protocol.map=INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL +log.dirs=./data +``` + +**2.4.3 Start Kafka brokers** + +Create a script, and then execute it to start the Kafka broker in each broker node. + +```shell +#!/bin/bash + +# Get the directory of the current script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Set JAVA_HOME to the Java installation within the script directory +export JAVA_HOME="$SCRIPT_DIR/jdk-22.0.2" +# Define the vars +KAFKA_DIR="$SCRIPT_DIR/kafka_2.13-3.7.1/bin" +KAFKA_STORAGE_CMD=$KAFKA_DIR/kafka-storage.sh +KAFKA_START_CMD=$KAFKA_DIR/kafka-server-start.sh +KAFKA_DATA_DIR=$SCRIPT_DIR/data +KAFKA_LOG_DIR=$SCRIPT_DIR/log +KAFKA_CONFIG_DIR=$SCRIPT_DIR/config + +# Cleanup step, which makes it easy for multiple experiments +# Find all Kafka process IDs +KAFKA_PIDS=$(ps aux | grep 'kafka.Kafka' | grep -v grep | awk '{print $2}') +if [ -z "$KAFKA_PIDS" ]; then + echo "No Kafka processes are running." +else + # Kill each Kafka process + echo "Killing Kafka processes with PIDs: $KAFKA_PIDS" + for PID in $KAFKA_PIDS; do + kill -9 $PID + echo "Killed Kafka process with PID: $PID" + done + echo "All Kafka processes have been killed." +fi + +rm -rf $KAFKA_DATA_DIR +mkdir -p $KAFKA_DATA_DIR +rm -rf $KAFKA_LOG_DIR +mkdir -p $KAFKA_LOG_DIR + +# Magic id: BRl69zcmTFmiPaoaANybiw, you can use your own +$KAFKA_STORAGE_CMD format -t "BRl69zcmTFmiPaoaANybiw" -c "$KAFKA_CONFIG_DIR/server.properties" > $KAFKA_LOG_DIR/server_format.log +LOG_DIR=$KAFKA_LOG_DIR nohup $KAFKA_START_CMD "$KAFKA_CONFIG_DIR/server.properties" & +``` + +**2.5. Test the cluster setting in the bastion node** + +1. Test the Kafka bootstrap. + + ```shell + export JAVA_HOME=/home/ec2-user/jdk-22.0.2 + + # Bootstrap from INTERNAL listener + ./kafka_2.13-3.7.1/bin/kafka-broker-api-versions.sh --bootstrap-server {one_of_broker_ip}:9092 | grep 9092 + # Expected output (the actual order might be different) + {broker-node1-ip}:9092 (id: 1 rack: null) -> ( + {broker-node2-ip}:9092 (id: 2 rack: null) -> ( + {broker-node3-ip}:9092 (id: 3 rack: null) -> ( + + # Bootstrap from EXTERNAL listener + ./kafka_2.13-3.7.1/bin/kafka-broker-api-versions.sh --bootstrap-server {one_of_broker_ip}:39092 + # Expected output for the last 3 lines (the actual order might be different) + # The difference in the output from "bootstrap from INTERNAL listener" is that exceptions or errors might occur because advertised listeners cannot be resolved in Kafka VPC. + # We will make them resolvable in TiDB Cloud side and make it route to the right broker when you create a changefeed connect to this Kafka cluster by Private Link. + b1.usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:9093 (id: 1 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + b2.usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:9094 (id: 2 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + b3.usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:9095 (id: 3 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + ``` + +2. Create a producer script `produce.sh` in the bastion node. + + ```shell + #!/bin/bash + BROKER_LIST=$1 # "{broker_address1},{broker_address2}..." + + # Get the directory of the current script + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + # Set JAVA_HOME to the Java installation within the script directory + export JAVA_HOME="$SCRIPT_DIR/jdk-22.0.2" + # Define the Kafka directory + KAFKA_DIR="$SCRIPT_DIR/kafka_2.13-3.7.1/bin" + TOPIC="test-topic" + + # Create a topic if it does not exist + create_topic() { + echo "Creating topic if it does not exist..." + $KAFKA_DIR/kafka-topics.sh --create --topic $TOPIC --bootstrap-server $BROKER_LIST --if-not-exists --partitions 3 --replication-factor 3 + } + + # Produce messages to the topic + produce_messages() { + echo "Producing messages to the topic..." + for ((chrono=1; chrono <= 10; chrono++)); do + message="Test message "$chrono + echo "Create "$message + echo $message | $KAFKA_DIR/kafka-console-producer.sh --broker-list $BROKER_LIST --topic $TOPIC + done + } + create_topic + produce_messages + ``` + +3. Create a consumer script `consume.sh` in the bastion node. + + ```shell + #!/bin/bash + + BROKER_LIST=$1 # "{broker_address1},{broker_address2}..." + + # Get the directory of the current script + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + # Set JAVA_HOME to the Java installation within the script directory + export JAVA_HOME="$SCRIPT_DIR/jdk-22.0.2" + # Define the Kafka directory + KAFKA_DIR="$SCRIPT_DIR/kafka_2.13-3.7.1/bin" + TOPIC="test-topic" + CONSUMER_GROUP="test-group" + # Consume messages from the topic + consume_messages() { + echo "Consuming messages from the topic..." + $KAFKA_DIR/kafka-console-consumer.sh --bootstrap-server $BROKER_LIST --topic $TOPIC --from-beginning --timeout-ms 5000 --consumer-property group.id=$CONSUMER_GROUP + } + consume_messages + ``` + +4. Execute `produce.sh` and `consume.sh` to verify that the Kafka cluster is running. These scripts will also be reused for later network connection testing. The script will create a topic with `--partitions 3 --replication-factor 3`. Ensure that all these three brokers contain data. Ensure that the script will connect to all three brokers to guarantee that network connection will be tested. + + ```shell + # Test write message. + ./produce.sh {one_of_broker_ip}:9092 + ``` + + ```shell + # Expected output + Creating topic if it does not exist... + + Producing messages to the topic... + Create Test message 1 + >>Create Test message 2 + >>Create Test message 3 + >>Create Test message 4 + >>Create Test message 5 + >>Create Test message 6 + >>Create Test message 7 + >>Create Test message 8 + >>Create Test message 9 + >>Create Test message 10 + ``` + + ```shell + # Test read message + ./consume.sh {one_of_broker_ip}:9092 + ``` + + ```shell + # Expected example output (the actual message order might be different) + Consuming messages from the topic... + Test message 3 + Test message 4 + Test message 5 + Test message 9 + Test message 10 + Test message 6 + Test message 8 + Test message 1 + Test message 2 + Test message 7 + [2024-11-01 08:54:27,547] ERROR Error processing message, terminating consumer process: (kafka.tools.ConsoleConsumer$) + org.apache.kafka.common.errors.TimeoutException + Processed a total of 10 messages + ``` + +### Reconfigure a running Kafka cluster + +Ensure that your Kafka cluster is deployed in the same region and AZs as the TiDB cluster. If any brokers are in different AZs, move them to the correct ones. + +#### 1. Configure the EXTERNAL listener for brokers + +The following configuration applies to a Kafka KRaft cluster. The ZK mode configuration is similar. + +1. Plan configuration changes. + + 1. Configure an EXTERNAL **listener** for every broker for external access from TiDB Cloud. Select a unique port as the EXTERNAL port, for example, `39092`. + 2. Configure an EXTERNAL **advertised listener** based on **Kafka Advertised Listener Pattern** you get from TiDB Cloud for every broker node to help TiDB Cloud differentiate between different brokers. Different EXTERNAL advertised listeners help Kafka clients from TiDB Cloud route requests to the right broker. + + - `` differentiates brokers from Kafka Private Link Service access points. Plan a port range for EXTERNAL advertised listeners of all brokers, for example, `range from 9093`. These ports do not have to be actual ports listened to by brokers. They are ports listened to by the load balancer for Private Link Service that will forward requests to different brokers. + - `AZ ID` in **Kafka Advertised Listener Pattern** indicates where the broker is deployed. TiDB Cloud will route requests to different endpoint DNS names based on the AZ ID. + + It is recommended to configure different broker IDs for different brokers to make it easy for troubleshooting. + +2. Use SSH to log in to each broker node. Modify the configuration file of each broker with the following content: + + ```properties + # brokers in usw2-az1 + + # Add EXTERNAL listener + listeners=INTERNAL:...,EXTERNAL://0.0.0.0:39092 + + # Add EXTERNAL advertised listeners based on the "Kafka Advertised Listener Pattern" in "Prerequisites" section + # 1. The pattern for AZ(ID: usw2-az1) is ".usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:" + # 2. So the EXTERNAL can be "b1.usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:9093", replace with "b" prefix plus "node.id" properties, replace with a unique port(9093) in EXTERNAL advertised listener ports range + advertised.listeners=...,EXTERNAL://b1.usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:9093 + + # Configure EXTERNAL map + listener.security.protocol.map=...,EXTERNAL:PLAINTEXT + ``` + + ```properties + # brokers in usw2-az2 + + # Add EXTERNAL listener + listeners=INTERNAL:...,EXTERNAL://0.0.0.0:39092 + + # Add EXTERNAL advertised listeners based on the "Kafka Advertised Listener Pattern" in "Prerequisites" section + # 1. The pattern for AZ(ID: usw2-az2) is ".usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:" + # 2. So the EXTERNAL can be "b2.usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:9094". Replace with "b" prefix plus "node.id" properties, and replace with a unique port(9094) in EXTERNAL advertised listener ports range. + advertised.listeners=...,EXTERNAL://b2.usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:9094 + + # Configure EXTERNAL map + listener.security.protocol.map=...,EXTERNAL:PLAINTEXT + ``` + + ```properties + # brokers in usw2-az3 + + # Add EXTERNAL listener + listeners=INTERNAL:...,EXTERNAL://0.0.0.0:39092 + + # Add EXTERNAL advertised listeners based on the "Kafka Advertised Listener Pattern" in "Prerequisites" section + # 1. The pattern for AZ(ID: usw2-az3) is ".usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:" + # 2. So the EXTERNAL can be "b2.usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:9095". Replace with "b" prefix plus "node.id" properties, and replace with a unique port(9095) in EXTERNAL advertised listener ports range. + advertised.listeners=...,EXTERNAL://b3.usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:9095 + + # Configure EXTERNAL map + listener.security.protocol.map=...,EXTERNAL:PLAINTEXT + ``` + +3. After you reconfigure all the brokers, restart your Kafka brokers one by one. + +#### 2. Test EXTERNAL listener settings in your internal network + +You can download the Kafka and OpenJDK in you Kafka client node. + +```shell +# Download Kafka and OpenJDK, and then extract the files. You can choose the binary version based on your preference. +wget https://downloads.apache.org/kafka/3.7.1/kafka_2.13-3.7.1.tgz +tar -zxf kafka_2.13-3.7.1.tgz +wget https://download.java.net/java/GA/jdk22.0.2/c9ecb94cd31b495da20a27d4581645e8/9/GPL/openjdk-22.0.2_linux-x64_bin.tar.gz +tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz +``` + +Execute the following script to test if the bootstrap works as expected. + +```shell +export JAVA_HOME=/home/ec2-user/jdk-22.0.2 + +# Bootstrap from the EXTERNAL listener +./kafka_2.13-3.7.1/bin/kafka-broker-api-versions.sh --bootstrap-server {one_of_broker_ip}:39092 + +# Expected output for the last 3 lines (the actual order might be different) +# There will be some exceptions or errors becasue advertised listeners cannot be resolved in your Kafka network. +# We will make them resolvable in TiDB Cloud side and make it route to the right broker when you create a changefeed connect to this Kafka cluster by Private Link. +b1.usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:9093 (id: 1 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException +b2.usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:9094 (id: 2 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException +b3.usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:9095 (id: 3 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException +``` + +## Step 2. Expose the Kafka cluster as Private Link Service + +### 1. Set up the load balancer + +Create a network load balancer with four target groups with different ports. One target group is for bootstrap, and the others will map to different brokers. + +1. bootstrap target group => 9092 => broker-node1:39092,broker-node2:39092,broker-node3:39092 +2. broker target group 1 => 9093 => broker-node1:39092 +3. broker target group 2 => 9094 => broker-node2:39092 +4. broker target group 3 => 9095 => broker-node3:39092 + +If you have more broker role nodes, you need to add more mappings. Ensure that you have at least one node in the bootstrap target group. It is recommended to add three nodes, one for each AZ for resilience. + +Do the following to set up the load balancer: + +1. Go to [Target groups](https://console.aws.amazon.com/ec2/home#CreateTargetGroup:) to create four target groups. + + - Bootstrap target group + + - **Target type**: `Instances` + - **Target group name**: `bootstrap-target-group` + - **Protocol**: `TCP` + - **Port**: `9092` + - **IP address type**: `IPv4` + - **VPC**: `Kafka VPC` + - **Health check protocol**: `TCP` + - **Register targets**: `broker-node1:39092`, `broker-node2:39092`, `broker-node3:39092` + + - Broker target group 1 + + - **Target type**: `Instances` + - **Target group name**: `broker-target-group-1` + - **Protocol**: `TCP` + - **Port**: `9093` + - **IP address type**: `IPv4` + - **VPC**: `Kafka VPC` + - **Health check protocol**: `TCP` + - **Register targets**: `broker-node1:39092` + + - Broker target group 2 + + - **Target type**: `Instances` + - **Target group name**: `broker-target-group-2` + - **Protocol**: `TCP` + - **Port**: `9094` + - **IP address type**: `IPv4` + - **VPC**: `Kafka VPC` + - **Health check protocol**: `TCP` + - **Register targets**: `broker-node2:39092` + + - Broker target group 3 + + - **Target type**: `Instances` + - **Target group name**: `broker-target-group-3` + - **Protocol**: `TCP` + - **Port**: `9095` + - **IP address type**: `IPv4` + - **VPC**: `Kafka VPC` + - **Health check protocol**: `TCP` + - **Register targets**: `broker-node3:39092` + +2. Go to [Load balancers](https://console.aws.amazon.com/ec2/home#LoadBalancers:) to create a network load balancer. + + - **Load balancer name**: `kafka-lb` + - **Schema**: `Internal` + - **Load balancer IP address type**: `IPv4` + - **VPC**: `Kafka VPC` + - **Availability Zones**: + - `usw2-az1` with `broker-usw2-az1 subnet` + - `usw2-az2` with `broker-usw2-az2 subnet` + - `usw2-az3` with `broker-usw2-az3 subnet` + - **Security groups**: create a new security group with the following rules. + - Inbound rule allows all TCP from Kafka VPC: Type - `All TCP`; Source - `Anywhere-IPv4` + - Outbound rule allows all TCP to Kafka VPC: Type - `All TCP`; Destination - `Anywhere-IPv4` + - Listeners and routing: + - Protocol: `TCP`; Port: `9092`; Forward to: `bootstrap-target-group` + - Protocol: `TCP`; Port: `9093`; Forward to: `broker-target-group-1` + - Protocol: `TCP`; Port: `9094`; Forward to: `broker-target-group-2` + - Protocol: `TCP`; Port: `9095`; Forward to: `broker-target-group-3` + +3. Test the load balancer in the bastion node. This example only tests the Kafka bootstrap. Because the load balancer is listening on the Kafka EXTERNAL listener, the addresses of EXTERNAL advertised listeners can not be resolved in the bastion node. Note down the `kafka-lb` DNS name from the load balancer detail page, for example `kafka-lb-77405fa57191adcb.elb.us-west-2.amazonaws.com`. Execute the script in the bastion node. + + ```shell + # Replace {lb_dns_name} to your actual value + export JAVA_HOME=/home/ec2-user/jdk-22.0.2 + ./kafka_2.13-3.7.1/bin/kafka-broker-api-versions.sh --bootstrap-server {lb_dns_name}:9092 + + # Expected output for the last 3 lines (the actual order might be different) + b1.usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:9093 (id: 1 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + b2.usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:9094 (id: 2 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + b3.usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:9095 (id: 3 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + + # You can also try bootstrap in other ports 9093/9094/9095. It will succeed probabilistically because NLB in AWS resolves LB DNS to the IP address of any availability zone and disables cross-zone load balancing by default. + # If you enable cross-zone load balancing in LB, it will succeed. However, it is unnecessary and might cause additional cross-AZ traffic. + ``` + +### 2. Set up Private Link Service + +1. Go to [Endpoint service](https://console.aws.amazon.com/vpcconsole/home#EndpointServices:). Click **Create endpoint service** to create a Private Link service for the Kafka load balancer. + + - **Name**: `kafka-pl-service` + - **Load balancer type**: `Network` + - **Load balancers**: `kafka-lb` + - **Included Availability Zones**: `usw2-az1`,`usw2-az2`, `usw2-az3` + - **Require acceptance for endpoint**: `Acceptance required` + - **Enable private DNS name**: `No` + +2. Note down the **Service name**. You need to provide it to TiDB Cloud, for example `com.amazonaws.vpce.us-west-2.vpce-svc-0f49e37e1f022cd45`. + +3. On the detail page of the kafka-pl-service, click the **Allow principals** tab, and allow the AWS account of TiDB Cloud to create the endpoint. You can get the AWS account of TiDB Cloud in [Prerequistes](#prerequisites), for example, `arn:aws:iam:::root`. + +## Step 3. Connect from TiDB Cloud + +1. Return to the [TiDB Cloud console](https://tidbcloud.com) to create a changefeed for the cluster to connect to the Kafka cluster by **Private Link**. For more information, see [Sink to Apache Kafka](/tidb-cloud/changefeed-sink-to-apache-kafka.md). + +2. When you proceed to **Configure the changefeed target > Connectivity Method > Private Link**, fill in the following fields with corresponding values and other fields as needed. + + - **Kafka Type**: `3 AZs`. Ensure that your Kafka cluster is deployed in the same three AZs. + - **Kafka Advertised Listener Pattern**: `abc`. It is the same as the unique random string you use to generate **Kafka Advertised Listener Pattern** in [Prerequistes](#prerequisites). + - **Endpoint Service Name**: the Kafka service name. + - **Bootstrap Ports**: `9092`. A single port is sufficient because you configure a dedicated bootstrap target group behind it. + +3. Proceed with the steps in [Sink to Apache Kafka](/tidb-cloud/changefeed-sink-to-apache-kafka.md). + +Now you have successfully finished the task. + +## FAQ + +### How to connect to the same Kafka Private Link service from two different TiDB Cloud projects? + +If you have already followed this document to successfully set up the connection from the first project, you can connect to the same Kafka Private Link service from the second project as follows: + +1. Follow instructions from the beginning of this document. + +2. When you proceed to [Step 1. Set up a Kafka cluster](#step-1-set-up-a-kafka-cluster), follow [Reconfigure a running Kafka cluster](#reconfigure-a-running-kafka-cluster) to create another group of EXTERNAL listeners and advertised listeners. You can name it as **EXTERNAL2**. Note that the port range of **EXTERNAL2** cannot overlap with the **EXTERNAL**. + +3. After reconfiguring brokers, add another target group in the load balancer, including the bootstrap and broker target groups. + +4. Configure the TiDB Cloud connection with the following information: + + - New Bootstrap port + - New Kafka Advertised Listener Group + - The same Endpoint Service diff --git a/tidb-cloud/setup-self-hosted-kafka-private-service-connect.md b/tidb-cloud/setup-self-hosted-kafka-private-service-connect.md new file mode 100644 index 0000000000000..16c1a4ac7e378 --- /dev/null +++ b/tidb-cloud/setup-self-hosted-kafka-private-service-connect.md @@ -0,0 +1,704 @@ +--- +title: Set Up Self-Hosted Kafka Private Service Connect in Google Cloud +summary: This document explains how to set up Private Service Connect for self-hosted Kafka in Google Cloud and how to make it work with TiDB Cloud. +--- + +# Set Up Self-Hosted Kafka Private Service Connect in Google Cloud + +This document explains how to set up Private Service Connect for self-hosted Kafka in Google Cloud, and how to make it work with TiDB Cloud. + +The mechanism works as follows: + +1. The TiDB Cloud VPC connects to the Kafka VPC through private endpoints. +2. Kafka clients need to communicate directly to all Kafka brokers. +3. Each Kafka broker is mapped to a unique port within the TiDB Cloud VPC. +4. Leverage the Kafka bootstrap mechanism and Google Cloud resources to achieve the mapping. + +There are two ways to set up Private Service Connect for self-hosted Kafka in Google Cloud: + +- Use the Private Service Connect (PSC) port mapping mechanism. This method requires static port-broker mapping configuration. You need to reconfigure the existing Kafka cluster to add a group of EXTERNAL listeners and advertised listeners. See [Set up self-hosted Kafka Private Service Connect service by PSC port mapping](#set-up-self-hosted-kafka-private-service-connect-service-by-psc-port-mapping). + +- Use [Kafka-proxy](https://github.com/grepplabs/kafka-proxy). This method introduces an extra running process as the proxy between Kafka clients and Kafka brokers. The proxy dynamically configures the port-broker mapping and forwards requests. You do not need to reconfigure the existing Kafka cluster. See [Set up self-hosted Kafka Private Service Connect by Kafka-proxy](#set-up-self-hosted-kafka-private-service-connect-by-kafka-proxy). + +The document provides an example of connecting to a Kafka Private Service Connect service deployed across three availability zones (AZ) in Google Cloud. While other configurations are possible based on similar port-mapping principles, this document covers the fundamental setup process of the Kafka Private Service Connect service. For production environments, a more resilient Kafka Private Service Connect service with enhanced operational maintainability and observability is recommended. + +## Prerequisites + +1. Ensure that you have the following authorization to set up Kafka Private Service Connect in your own Google Cloud account. + + - Manage VM nodes + - Manage VPC + - Manage subnets + - Manage load balancer + - Manage Private Service Connect + - Connect to VM nodes to configure Kafka nodes + +2. [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md) if you do not have one. + +3. Get the Kafka deployment information from your TiDB Cloud Dedicated cluster. + + 1. In the [TiDB Cloud console](https://tidbcloud.com), navigate to the cluster overview page of the TiDB cluster, and then click **Changefeed** in the left navigation pane. + 2. On the overview page, find the region of the TiDB cluster. Ensure that your Kafka cluster will be deployed to the same region. + 3. Click **Create Changefeed**. + 1. In **Target Type**, select **Kafka**. + 2. In **Connectivity Method**, select **Private Service Connect**. + 4. Note down the Google Cloud project in **Reminders before proceeding**. You will use it to authorize the auto-accept endpoint creation request from TiDB Cloud. + 5. Note down the **Zones of TiDB Cluster**. You will deploy your TiDB cluster in these zones. It is recommended that you deploy Kafka in these zones to reduce cross-zone traffic. + 6. Pick a unique **Kafka Advertised Listener Pattern** for your Kafka Private Service Connect service. + 1. Input a unique random string. It can only include numbers or lowercase letters. You will use it to generate **Kafka Advertised Listener Pattern** later. + 2. Click **Check usage and generate** to check if the random string is unique and generate **Kafka Advertised Listener Pattern** that will be used to assemble the EXTERNAL advertised listener for Kafka brokers, or configure Kafka-proxy. + +Note down all the deployment information. You need to use it to configure your Kafka Private Service Connect service later. + +The following table shows an example of the deployment information. + +| Information | Value | +|------------------------------------|---------------------------------| +| Region | Oregon (`us-west1`) | +| Google Cloud project of TiDB Cloud | `tidbcloud-prod-000` | +| Zones |
    • `us-west1-a`
    • `us-west1-b`
    • `us-west1-c`
    | +| Kafka Advertised Listener Pattern | The unique random string: `abc`
    Generated pattern: <broker_id>.abc.us-west1.gcp.3199745.tidbcloud.com:<port> | + +## Set up self-hosted Kafka Private Service Connect service by PSC port mapping + +Expose each Kafka broker to TiDB Cloud VPC with a unique port by using the PSC port mapping mechanism. The following diagram shows how it works. + +![Connect to Google Cloud self-hosted Kafka Private Service Connect by port mapping](/media/tidb-cloud/changefeed/connect-to-google-cloud-self-hosted-kafka-private-service-connect-by-portmapping.jpeg) + +### Step 1. Set up the Kafka cluster + +If you need to deploy a new cluster, follow the instructions in [Deploy a new Kafka cluster](#deploy-a-new-kafka-cluster). + +If you need to expose an existing cluster, follow the instructions in [Reconfigure a running Kafka cluster](#reconfigure-a-running-kafka-cluster). + +#### Deploy a new Kafka cluster + +**1. Set up the Kafka VPC** + +You need to create two subnets for Kafka VPC, one for Kafka brokers, and the other for the bastion node to make it easy to configure the Kafka cluster. + +Go to the [Google Cloud console](https://cloud.google.com/cloud-console), and navigate to the [VPC networks](https://console.cloud.google.com/networking/networks/list) page to create the Kafka VPC with the following attributes: + +- **Name**: `kafka-vpc` +- Subnets + - **Name**: `bastion-subnet`; **Region**: `us-west1`; **IPv4 range**: `10.0.0.0/18` + - **Name**: `brokers-subnet`; **Region**: `us-west1`; **IPv4 range**: `10.64.0.0/18` +- Firewall rules + - `kafka-vpc-allow-custom` + - `kafka-vpc-allow-ssh` + +**2. Provisioning VMs** + +Go to the [VM instances](https://console.cloud.google.com/compute/instances) page to provision VMs: + +1. Bastion node + + - **Name**: `bastion-node` + - **Region**: `us-west1` + - **Zone**: `Any` + - **Machine Type**: `e2-medium` + - **Image**: `Debian GNU/Linux 12` + - **Network**: `kafka-vpc` + - **Subnetwork**: `bastion-subnet` + - **External IPv4 address**: `Ephemeral` + +2. Broker node 1 + + - **Name**: `broker-node1` + - **Region**: `us-west1` + - **Zone**: `us-west1-a` + - **Machine Type**: `e2-medium` + - **Image**: `Debian GNU/Linux 12` + - **Network**: `kafka-vpc` + - **Subnetwork**: `brokers-subnet` + - **External IPv4 address**: `None` + +3. Broker node 2 + + - **Name**: `broker-node2` + - **Region**: `us-west1` + - **Zone**: `us-west1-b` + - **Machine Type**: `e2-medium` + - **Image**: `Debian GNU/Linux 12` + - **Network**: `kafka-vpc` + - **Subnetwork**: `brokers-subnet` + - **External IPv4 address**: `None` + +4. Broker node 3 + + - **Name**: `broker-node3` + - **Region**: `us-west1` + - **Zone**: `us-west1-c` + - **Machine Type**: `e2-medium` + - **Image**: `Debian GNU/Linux 12` + - **Network**: `kafka-vpc` + - **Subnetwork**: `brokers-subnet` + - **External IPv4 address**: `None` + +**3. Prepare Kafka runtime binaries** + +1. Go to the detail page of the bastion node. Click **SSH** to log in to the bastion node. Download binaries. + + ```shell + # Download Kafka and OpenJDK, and then extract the files. You can choose the binary version based on your preference. + wget https://downloads.apache.org/kafka/3.7.1/kafka_2.13-3.7.1.tgz + tar -zxf kafka_2.13-3.7.1.tgz + wget https://download.java.net/java/GA/jdk22.0.2/c9ecb94cd31b495da20a27d4581645e8/9/GPL/openjdk-22.0.2_linux-x64_bin.tar.gz + tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz + ``` + +2. Copy binaries to each broker node. + + ```shell + # Run this command to authorize gcloud to access the Cloud Platform with Google user credentials + # Follow the instruction in output to finish the login + gcloud auth login + + # Copy binaries to broker nodes + gcloud compute scp kafka_2.13-3.7.1.tgz openjdk-22.0.2_linux-x64_bin.tar.gz broker-node1:~ --zone=us-west1-a + gcloud compute ssh broker-node1 --zone=us-west1-a --command="tar -zxf kafka_2.13-3.7.1.tgz && tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz" + gcloud compute scp kafka_2.13-3.7.1.tgz openjdk-22.0.2_linux-x64_bin.tar.gz broker-node2:~ --zone=us-west1-b + gcloud compute ssh broker-node2 --zone=us-west1-b --command="tar -zxf kafka_2.13-3.7.1.tgz && tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz" + gcloud compute scp kafka_2.13-3.7.1.tgz openjdk-22.0.2_linux-x64_bin.tar.gz broker-node3:~ --zone=us-west1-c + gcloud compute ssh broker-node3 --zone=us-west1-c --command="tar -zxf kafka_2.13-3.7.1.tgz && tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz" + ``` + +**4. Configure Kafka brokers** + +1. Set up a KRaft Kafka cluster with three nodes. Each node acts as a broker and controller roles. For every broker: + + 1. For `listeners`, all three brokers are the same and act as brokers and controller roles: + 1. Configure the same CONTROLLER listener for all **controller** role nodes. If you only want to add the **broker** role nodes, you do not need the CONTROLLER listener in `server.properties`. + 2. Configure two **broker** listeners. INTERNAL for internal access; EXTERNAL for external access from TiDB Cloud. + + 2. For `advertised.listeners`, do the following: + 1. Configure an INTERNAL advertised listener for each broker using the internal IP address of the broker node, which allows internal Kafka clients to connect to the broker via the advertised address. + 2. Configure an EXTERNAL advertised listener based on **Kafka Advertised Listener Pattern** you get from TiDB Cloud for every broker node to help TiDB Cloud differentiate between different brokers. Different EXTERNAL advertised listeners help Kafka clients from TiDB Cloud side route requests to the right broker. + - `` differentiates brokers from Kafka Private Service Connect access points. Plan a port range for EXTERNAL advertised listeners of all brokers. These ports do not have to be actual ports listened to by brokers. They are ports listened to by the load balancer for Private Service Connect that will forward requests to different brokers. + - It is recommended to configure different broker IDs for different brokers to make it easy for troubleshooting. + + 3. The planning values: + - CONTROLLER port: `29092` + - INTERNAL port: `9092` + - EXTERNAL: `39092` + - EXTERNAL advertised listener ports range: `9093~9095` + +2. Use SSH to log in to every broker node. Create a configuration file `~/config/server.properties` with the following content for each broker node respectively. + + ```properties + # broker-node1 ~/config/server.properties + # 1. Replace {broker-node1-ip}, {broker-node2-ip}, {broker-node3-ip} with the actual IP addresses. + # 2. Configure EXTERNAL in "advertised.listeners" based on the "Kafka Advertised Listener Pattern" in the "Prerequisites" section. + # 2.1 The pattern is ".abc.us-west1.gcp.3199745.tidbcloud.com:". + # 2.2 So the EXTERNAL can be "b1.abc.us-west1.gcp.3199745.tidbcloud.com:9093". Replace with "b" prefix plus "node.id" properties, and replace with a unique port (9093) in the port range of the EXTERNAL advertised listener. + process.roles=broker,controller + node.id=1 + controller.quorum.voters=1@{broker-node1-ip}:29092,2@{broker-node2-ip}:29092,3@{broker-node3-ip}:29092 + listeners=INTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29092,EXTERNAL://0.0.0.0:39092 + inter.broker.listener.name=INTERNAL + advertised.listeners=INTERNAL://{broker-node1-ip}:9092,EXTERNAL://b1.abc.us-west1.gcp.3199745.tidbcloud.com:9093 + controller.listener.names=CONTROLLER + listener.security.protocol.map=INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL + log.dirs=./data + ``` + + ```properties + # broker-node2 ~/config/server.properties + # 1. Replace {broker-node1-ip}, {broker-node2-ip}, {broker-node3-ip} with the actual IP addresses. + # 2. Configure EXTERNAL in "advertised.listeners" based on the "Kafka Advertised Listener Pattern" in the "Prerequisites" section. + # 2.1 The pattern is ".abc.us-west1.gcp.3199745.tidbcloud.com:". + # 2.2 So the EXTERNAL can be "b2.abc.us-west1.gcp.3199745.tidbcloud.com:9094". Replace with "b" prefix plus "node.id" properties, and replace with a unique port (9094) in the port range of the EXTERNAL advertised listener. + process.roles=broker,controller + node.id=2 + controller.quorum.voters=1@{broker-node1-ip}:29092,2@{broker-node2-ip}:29092,3@{broker-node3-ip}:29092 + listeners=INTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29092,EXTERNAL://0.0.0.0:39092 + inter.broker.listener.name=INTERNAL + advertised.listeners=INTERNAL://{broker-node2-ip}:9092,EXTERNAL://b2.abc.us-west1.gcp.3199745.tidbcloud.com:9094 + controller.listener.names=CONTROLLER + listener.security.protocol.map=INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL + log.dirs=./data + ``` + + ```properties + # broker-node3 ~/config/server.properties + # 1. Replace {broker-node1-ip}, {broker-node2-ip}, {broker-node3-ip} with the actual IP addresses. + # 2. Configure EXTERNAL in "advertised.listeners" based on the "Kafka Advertised Listener Pattern" in the "Prerequisites" section. + # 2.1 The pattern is ".abc.us-west1.gcp.3199745.tidbcloud.com:". + # 2.2 So the EXTERNAL can be "b3.abc.us-west1.gcp.3199745.tidbcloud.com:9095". Replace with "b" prefix plus "node.id" properties, and replace with a unique port (9095) in the port range of the EXTERNAL advertised listener. + process.roles=broker,controller + node.id=3 + controller.quorum.voters=1@{broker-node1-ip}:29092,2@{broker-node2-ip}:29092,3@{broker-node3-ip}:29092 + listeners=INTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29092,EXTERNAL://0.0.0.0:39092 + inter.broker.listener.name=INTERNAL + advertised.listeners=INTERNAL://{broker-node3-ip}:9092,EXTERNAL://b3.abc.us-west1.gcp.3199745.tidbcloud.com:9095 + controller.listener.names=CONTROLLER + listener.security.protocol.map=INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL + log.dirs=./data + ``` + +3. Create a script, and then execute it to start the Kafka broker in each broker node. + + ```shell + #!/bin/bash + + # Get the directory of the current script + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + # Set JAVA_HOME to the Java installation within the script directory + export JAVA_HOME="$SCRIPT_DIR/jdk-22.0.2" + # Define the vars + KAFKA_DIR="$SCRIPT_DIR/kafka_2.13-3.7.1/bin" + KAFKA_STORAGE_CMD=$KAFKA_DIR/kafka-storage.sh + KAFKA_START_CMD=$KAFKA_DIR/kafka-server-start.sh + KAFKA_DATA_DIR=$SCRIPT_DIR/data + KAFKA_LOG_DIR=$SCRIPT_DIR/log + KAFKA_CONFIG_DIR=$SCRIPT_DIR/config + + # Cleanup step, which makes it easy for multiple experiments + # Find all Kafka process IDs + KAFKA_PIDS=$(ps aux | grep 'kafka.Kafka' | grep -v grep | awk '{print $2}') + if [ -z "$KAFKA_PIDS" ]; then + echo "No Kafka processes are running." + else + # Kill each Kafka process + echo "Killing Kafka processes with PIDs: $KAFKA_PIDS" + for PID in $KAFKA_PIDS; do + kill -9 $PID + echo "Killed Kafka process with PID: $PID" + done + echo "All Kafka processes have been killed." + fi + + rm -rf $KAFKA_DATA_DIR + mkdir -p $KAFKA_DATA_DIR + rm -rf $KAFKA_LOG_DIR + mkdir -p $KAFKA_LOG_DIR + + # Magic id: BRl69zcmTFmiPaoaANybiw. You can use your own magic ID. + $KAFKA_STORAGE_CMD format -t "BRl69zcmTFmiPaoaANybiw" -c "$KAFKA_CONFIG_DIR/server.properties" > $KAFKA_LOG_DIR/server_format.log + LOG_DIR=$KAFKA_LOG_DIR nohup $KAFKA_START_CMD "$KAFKA_CONFIG_DIR/server.properties" & + ``` + +**5. Test the Kafka cluster in the bastion node** + +1. Test the Kafka bootstrap. + + ```shell + export JAVA_HOME=~/jdk-22.0.2 + + # Bootstrap from INTERNAL listener + ./kafka_2.13-3.7.1/bin/kafka-broker-api-versions.sh --bootstrap-server {one_of_broker_ip}:9092 | grep 9092 + # Expected output (the actual order might be different) + {broker-node1-ip}:9092 (id: 1 rack: null) -> ( + {broker-node2-ip}:9092 (id: 2 rack: null) -> ( + {broker-node3-ip}:9092 (id: 3 rack: null) -> ( + + # Bootstrap from EXTERNAL listener + ./kafka_2.13-3.7.1/bin/kafka-broker-api-versions.sh --bootstrap-server {one_of_broker_ip}:39092 + # Expected output for the last 3 lines (the actual order might be different) + # The difference in the output from "bootstrap from INTERNAL listener" is that exceptions or errors might occur because advertised listeners cannot be resolved in Kafka VPC. + # We will make them resolvable in TiDB Cloud side and make it route to the right broker when you create a changefeed connect to this Kafka cluster by Private Service Connect. + b1.abc.us-west1.gcp.3199745.tidbcloud.com:9093 (id: 1 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + b2.abc.us-west1.gcp.3199745.tidbcloud.com:9094 (id: 2 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + b3.abc.us-west1.gcp.3199745.tidbcloud.com:9095 (id: 3 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + ``` + +2. Create a producer script `produce.sh` in the bastion node. + + ```shell + #!/bin/bash + BROKER_LIST=$1 # "{broker_address1},{broker_address2}..." + + # Get the directory of the current script + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + # Set JAVA_HOME to the Java installation within the script directory + export JAVA_HOME="$SCRIPT_DIR/jdk-22.0.2" + # Define the Kafka directory + KAFKA_DIR="$SCRIPT_DIR/kafka_2.13-3.7.1/bin" + TOPIC="test-topic" + + # Create a topic if it does not exist + create_topic() { + echo "Creating topic if it does not exist..." + $KAFKA_DIR/kafka-topics.sh --create --topic $TOPIC --bootstrap-server $BROKER_LIST --if-not-exists --partitions 3 --replication-factor 3 + } + + # Produce messages to the topic + produce_messages() { + echo "Producing messages to the topic..." + for ((chrono=1; chrono <= 10; chrono++)); do + message="Test message "$chrono + echo "Create "$message + echo $message | $KAFKA_DIR/kafka-console-producer.sh --broker-list $BROKER_LIST --topic $TOPIC + done + } + create_topic + produce_messages + ``` + +3. Create a consumer script `consume.sh` in the bastion node. + + ```shell + #!/bin/bash + + BROKER_LIST=$1 # "{broker_address1},{broker_address2}..." + + # Get the directory of the current script + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + # Set JAVA_HOME to the Java installation within the script directory + export JAVA_HOME="$SCRIPT_DIR/jdk-22.0.2" + # Define the Kafka directory + KAFKA_DIR="$SCRIPT_DIR/kafka_2.13-3.7.1/bin" + TOPIC="test-topic" + CONSUMER_GROUP="test-group" + # Consume messages from the topic + consume_messages() { + echo "Consuming messages from the topic..." + $KAFKA_DIR/kafka-console-consumer.sh --bootstrap-server $BROKER_LIST --topic $TOPIC --from-beginning --timeout-ms 5000 --consumer-property group.id=$CONSUMER_GROUP + } + consume_messages + ``` + +4. Execute `produce.sh` and `consume.sh` to verify that the Kafka cluster is running. These scripts will also be reused for later network connection testing. The script will create a topic with `--partitions 3 --replication-factor 3`. Ensure that all three brokers contain data. Ensure that the scripts will connect to all three brokers to guarantee that network connection will be tested. + + ```shell + # Test write message. + ./produce.sh {one_of_broker_ip}:9092 + ``` + + ```text + # Expected output + Creating topic if it does not exist... + + Producing messages to the topic... + Create Test message 1 + >>Create Test message 2 + >>Create Test message 3 + >>Create Test message 4 + >>Create Test message 5 + >>Create Test message 6 + >>Create Test message 7 + >>Create Test message 8 + >>Create Test message 9 + >>Create Test message 10 + ``` + + ```shell + # Test read message + ./consume.sh {one_of_broker_ip}:9092 + ``` + + ```text + # Expected example output (the actual message order might be different) + Consuming messages from the topic... + Test message 3 + Test message 4 + Test message 5 + Test message 9 + Test message 10 + Test message 6 + Test message 8 + Test message 1 + Test message 2 + Test message 7 + [2024-11-01 08:54:27,547] ERROR Error processing message, terminating consumer process: (kafka.tools.ConsoleConsumer$) + org.apache.kafka.common.errors.TimeoutException + Processed a total of 10 messages + ``` + +#### Reconfigure a running Kafka cluster + +Ensure that your Kafka cluster is deployed in the same region as the TiDB cluster. It is recommended that the zones are also in the same region to reduce cross-zone traffic. + +**1. Configure the EXTERNAL listener for brokers** + +The following configuration applies to a Kafka KRaft cluster. The ZK mode configuration is similar. + +1. Plan configuration changes. + + 1. Configure an EXTERNAL **listener** for every broker for external access from TiDB Cloud. Select a unique port as the EXTERNAL port, for example, `39092`. + 2. Configure an EXTERNAL **advertised listener** based on **Kafka Advertised Listener Pattern** you get from TiDB Cloud for every broker node to help TiDB Cloud differentiate between different brokers. Different EXTERNAL advertised listeners help Kafka clients from TiDB Cloud side route requests to the right broker. + - `` differentiates brokers from Kafka Private Service Connect access points. Plan a port range for EXTERNAL advertised listeners of all brokers, for example, `range from 9093`. These ports do not have to be actual ports listened to by brokers. They are ports listened to by the load balancer for Private Service Connect that will forward requests to different brokers. + - It is recommended to configure different broker IDs for different brokers to make it easy for troubleshooting. + +2. Use SSH to log in to each broker node. Modify the configuration file of each broker with the following content: + + ```properties + # Add EXTERNAL listener + listeners=INTERNAL:...,EXTERNAL://0.0.0.0:39092 + + # Add EXTERNAL advertised listeners based on the "Kafka Advertised Listener Pattern" in the "Prerequisites" section + # 1. The pattern is ".abc.us-west1.gcp.3199745.tidbcloud.com:". + # 2. So the EXTERNAL can be "bx.abc.us-west1.gcp.3199745.tidbcloud.com:xxxx". Replace with "b" prefix plus "node.id" properties, and replace with a unique port in the port range of the EXTERNAL advertised listener. + # For example + advertised.listeners=...,EXTERNAL://b1.abc.us-west1.gcp.3199745.tidbcloud.com:9093 + + # Configure EXTERNAL map + listener.security.protocol.map=...,EXTERNAL:PLAINTEXT + ``` + +3. After you reconfigure all the brokers, restart your Kafka brokers one by one. + +**2. Test EXTERNAL listener settings in your internal network** + +You can download Kafka and OpenJDK in your Kafka client node. + +```shell +# Download Kafka and OpenJDK, and then extract the files. You can choose the binary version based on your preference. +wget https://downloads.apache.org/kafka/3.7.1/kafka_2.13-3.7.1.tgz +tar -zxf kafka_2.13-3.7.1.tgz +wget https://download.java.net/java/GA/jdk22.0.2/c9ecb94cd31b495da20a27d4581645e8/9/GPL/openjdk-22.0.2_linux-x64_bin.tar.gz +tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz +``` + +Execute the following script to test if the bootstrap works as expected. + +```shell +export JAVA_HOME=~/jdk-22.0.2 + +# Bootstrap from the EXTERNAL listener +./kafka_2.13-3.7.1/bin/kafka-broker-api-versions.sh --bootstrap-server {one_of_broker_ip}:39092 + +# Expected output for the last 3 lines (the actual order might be different) +# There will be some exceptions or errors because advertised listeners cannot be resolved in your Kafka network. +# We will make them resolvable in TiDB Cloud side and make it route to the right broker when you create a changefeed connect to this Kafka cluster by Private Service Connect. +b1.abc.us-west1.gcp.3199745.tidbcloud.com:9093 (id: 1 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException +b2.abc.us-west1.gcp.3199745.tidbcloud.com:9094 (id: 2 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException +b3.abc.us-west1.gcp.3199745.tidbcloud.com:9095 (id: 3 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException +``` + +### Step 2. Expose the Kafka cluster as Private Service Connect + +1. Go to the [Network endpoint group](https://console.cloud.google.com/compute/networkendpointgroups/list) page. Create a network endpoint group as follows: + + - **Name**: `kafka-neg` + - **Network endpoint group type**: `Port Mapping NEG(Regional)` + - **Region**: `us-west1` + - **Network**: `kafka-vpc` + - **Subnet**: `brokers-subnet` + +2. Go to the detail page of the network endpoint group, and add the network endpoints to configure the port mapping to broker nodes. + + 1. Network endpoint 1 + - **Instance**: `broker-node1` + - **VM Port**: `39092` + - **Client Port**: `9093` + 2. Network endpoint 2 + - **Instance**: `broker-node2` + - **VM Port**: `39092` + - **Client Port**: `9094` + 3. Network endpoint 3 + - **Instance**: `broker-node3` + - **VM Port**: `39092` + - **Client Port**: `9095` + +3. Go to the [Load balancing](https://console.cloud.google.com/net-services/loadbalancing/list/loadBalancers) page. Create a load balancer as follows: + + - **Type of load balancer**: `Network Load Balancer` + - **Proxy or Passthrough**: `Passthrough` + - **Public facing or internal**: `Internal` + - **Load Balancer name**: `kafka-lb` + - **Region**: `us-west1` + - **Network**: `kafka-vpc` + - Backend configuration + - **Backend type**: `Port mapping network endpoint group` + - **Protocol**: `TCP` + - **Port mapping network endpoint group**: `kafka-neg` + - Frontend configuration + - **Subnetwork**: `brokers-subnet` + - **Ports**: `All` + +4. Go to [**Private Service Connect** > **PUBLISH SERVICE**](https://console.cloud.google.com/net-services/psc/list/producers). + + - **Load Balancer Type**: `Internal passthrough Network Load Balancer` + - **Internal load balancer**: `kafka-lb` + - **Service name**: `kafka-psc` + - **Subnets**: `RESERVE NEW SUBNET` + - **Name**: `psc-subnet` + - **VPC Network**: `kafka-vpc` + - **Region**: `us-west1` + - **IPv4 range**: `10.128.0.0/18` + - **Accepted projects**: the Google Cloud project of TiDB Cloud you got in [Prerequisites](#prerequisites), for example, `tidbcloud-prod-000`. + +5. Navigate to the detail page of the `kafka-psc`. Note down the **Service attachment**, for example, `projects/tidbcloud-dp-stg-000/regions/us-west1/serviceAttachments/kafka-psc`. You will use it in TiDB Cloud to connect to this PSC. + +6. Go to the detail page of the VPC network `kafka-vpc`. Add a firewall rule to allow PSC traffic to all brokers. + + - **Name**: `allow-psc-traffic` + - **Direction of traffic**: `Ingress` + - **Action on match**: `Allow` + - **Targets**: `All instances in the network` + - **Source filter**: `IPv4 ranges` + - **Source IPv4 ranges**: `10.128.0.0/18`. The range of psc-subnet. + - **Protocols and ports**: Allow all + +### Step 3. Connect from TiDB Cloud + +1. Go back to the [TiDB Cloud console](https://tidbcloud.com) to create a changefeed for the cluster to connect to the Kafka cluster by **Private Service Connect**. For more information, see [Sink to Apache Kafka](/tidb-cloud/changefeed-sink-to-apache-kafka.md). + +2. When you proceed to **Configure the changefeed target > Connectivity Method > Private Service Connect**, fill in the following fields with corresponding values and other fields as needed. + + - **Kafka Advertised Listener Pattern**: `abc`. It is the same as the unique random string you use to generate **Kafka Advertised Listener Pattern** in [Prerequisites](#prerequisites). + - **Service Attachment**: the Kafka service attachment of PSC, for example, `projects/tidbcloud-dp-stg-000/regions/us-west1/serviceAttachments/kafka-psc`. + - **Bootstrap Ports**: `9092,9093,9094` + +3. Proceed with the steps in [Sink to Apache Kafka](/tidb-cloud/changefeed-sink-to-apache-kafka.md). + +## Set up self-hosted Kafka Private Service Connect by Kafka-proxy + +Expose each Kafka broker to TiDB Cloud VPC with a unique port by using the Kafka-proxy dynamic port mapping mechanism. The following diagram shows how it works. + +![Connect to Google Cloud self-hosted Kafka Private Service Connect by Kafka proxy](/media/tidb-cloud/changefeed/connect-to-google-cloud-self-hosted-kafka-private-service-connect-by-kafka-proxy.jpeg) + +### Step 1. Set up Kafka-proxy + +Assume that you already have a Kafka cluster running in the same region as the TiDB cluster. You can connect to the Kafka cluster from your VPC network. The Kafka cluster can be self-hosted or provided by third-party providers, such as Confluent. + +1. Go to the [Instance groups](https://console.cloud.google.com/compute/instanceGroups/list) page, and create an instance group for Kafka-proxy. + + - **Name**: `kafka-proxy-ig` + - Instance template: + - **Name**: `kafka-proxy-tpl` + - **Location**: `Regional` + - **Region**: `us-west1` + - **Machine type**: `e2-medium`. You can choose your own machine type based on your workload. + - **Network**: your VPC network that can connect to the Kafka cluster. + - **Subnetwork**: your subnet that can connect to the Kafka cluster. + - **External IPv4 address**: `Ephemeral`. Enable Internet access to make it easy to configure Kafka-proxy. You can select **None** in your production environment and log in to the node in your way. + - **Location**: `Single zone` + - **Region**: `us-west1` + - **Zone**: choose one of your broker's zones. + - **Autoscaling mode**: `Off` + - **Minimum number of instances**: `1` + - **Maximum number of instances**: `1`. Kafka-proxy does not support the cluster mode, so only one instance can be deployed. Each Kafka-proxy randomly maps local ports to the ports of the broker, resulting in different mappings across proxies. Deploying multiple Kafka-proxies behind a load balancer can cause issues. If a Kafka client connects to one proxy and then accesses a broker through another, the request might be routed to the wrong broker. + +2. Go to the detail page of the node in kafka-proxy-ig. Click **SSH** to log in to the node. Download the binaries: + + ```shell + # You can choose another version + wget https://github.com/grepplabs/kafka-proxy/releases/download/v0.3.11/kafka-proxy-v0.3.11-linux-amd64.tar.gz + tar -zxf kafka-proxy-v0.3.11-linux-amd64.tar.gz + ``` + +3. Run Kafka-proxy and connect to Kafka brokers. + + ```shell + # There are three kinds of parameters that need to feed to the Kafka-proxy + # 1. --bootstrap-server-mapping defines the bootstrap mapping. Suggest that you configure three mappings, one for each zone for resilience. + # a) Kafka broker address; + # b) Local address for the broker in Kafka-proxy; + # c) Advertised listener for the broker if Kafka clients bootstrap from Kafka-proxy + # 2. --dynamic-sequential-min-port defines the start port of the random mapping for other brokers + # 3. --dynamic-advertised-listener defines advertised listener address for other brokers based on the pattern obtained from the "Prerequisites" section + # a) The pattern: .abc.us-west1.gcp.3199745.tidbcloud.com: + # b) Make sure to replace with a fixed lowercase string, for example, "brokers". You can use your own string. This step will help TiDB Cloud route requests properly. + # c) Remove ":" + # d) The advertised listener address would be: brokers.abc.us-west1.gcp.3199745.tidbcloud.com + ./kafka-proxy server \ + --bootstrap-server-mapping "{address_of_broker1},0.0.0.0:9092,b1.abc.us-west1.gcp.3199745.tidbcloud.com:9092" \ + --bootstrap-server-mapping "{address_of_broker2},0.0.0.0:9093,b2.abc.us-west1.gcp.3199745.tidbcloud.com:9093" \ + --bootstrap-server-mapping "{address_of_broker3},0.0.0.0:9094,b3.abc.us-west1.gcp.3199745.tidbcloud.com:9094" \ + --dynamic-sequential-min-port=9095 \ + --dynamic-advertised-listener=brokers.abc.us-west1.gcp.3199745.tidbcloud.com > ./kafka_proxy.log 2>&1 & + ``` + +4. Test bootstrap in the Kafka-proxy node. + + ```shell + # Download Kafka and OpenJDK, and then extract the files. You can choose the binary version based on your preference. + wget https://downloads.apache.org/kafka/3.7.1/kafka_2.13-3.7.1.tgz + tar -zxf kafka_2.13-3.7.1.tgz + wget https://download.java.net/java/GA/jdk22.0.2/c9ecb94cd31b495da20a27d4581645e8/9/GPL/openjdk-22.0.2_linux-x64_bin.tar.gz + tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz + + export JAVA_HOME=~/jdk-22.0.2 + + ./kafka_2.13-3.7.1/bin/kafka-broker-api-versions.sh --bootstrap-server 0.0.0.0:9092 + # Expected output of the last few lines (the actual order might be different) + # There might be exceptions or errors because advertised listeners cannot be resolved in your network. + # We will make them resolvable in TiDB Cloud side and make it route to the right broker when you create a changefeed connect to this Kafka cluster by Private Service Connect. + b1.abc.us-west1.gcp.3199745.tidbcloud.com:9092 (id: 1 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + b2.abc.us-west1.gcp.3199745.tidbcloud.com:9093 (id: 2 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + b3.abc.us-west1.gcp.3199745.tidbcloud.com:9094 (id: 3 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + brokers.abc.us-west1.gcp.3199745.tidbcloud.com:9095 (id: 4 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + brokers.abc.us-west1.gcp.3199745.tidbcloud.com:9096 (id: 5 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + ... + ``` + +### Step 2. Expose Kafka-proxy as Private Service Connect Service + +1. Go to the [Load balancing](https://console.cloud.google.com/net-services/loadbalancing/list/loadBalancers) page, and create a load balancer. + + - **Type of load balancer**: `Network Load Balancer` + - **Proxy or Passthrough**: `Passthrough` + - **Public facing or internal**: `Internal` + - **Load Balancer name**: `kafka-proxy-lb` + - **Region**: `us-west1` + - **Network**: your network + - Backend configuration + - **Backend type**: `Instance group` + - **Protocol**: `TCP` + - **Instance group**: `kafka-proxy-ig` + - Frontend configuration + - **Subnetwork**: your subnet + - **Ports**: `All` + - Heath check: + - **Name**: `kafka-proxy-hc` + - **Scope**: `Regional` + - **Protocol**: `TCP` + - **Port**: `9092`. You can select one of the bootstrap ports in Kafka-proxy. + +2. Go to [**Private Service Connect** > **PUBLISH SERVICE**](https://console.cloud.google.com/net-services/psc/list/producers). + + - **Load Balancer Type**: `Internal passthrough Network Load Balancer` + - **Internal load balancer**: `kafka-proxy-lb` + - **Service name**: `kafka-proxy-psc` + - **Subnets**: `RESERVE NEW SUBNET` + - **Name**: `proxy-psc-subnet` + - **VPC Network**: your network + - **Region**: `us-west1` + - **IPv4 range**: set the CIDR based on your network planning + - **Accepted projects**: the Google Cloud project of TiDB Cloud you get in [Prerequisites](#prerequisites), for example, `tidbcloud-prod-000`. + +3. Navigate to the detail page of the **kafka-proxy-psc**. Note down the `Service attachment`, for example, `projects/tidbcloud-dp-stg-000/regions/us-west1/serviceAttachments/kafka-proxy-psc`, which will be used in TiDB Cloud to connect to this PSC. + +4. Go to the detail page of your VPC network. Add a firewall rule to allow the PSC traffic for all brokers. + + - **Name**: `allow-proxy-psc-traffic` + - **Direction of traffic**: `Ingress` + - **Action on match**: `Allow` + - **Targets**: All instances in the network + - **Source filter**: `IPv4 ranges` + - **Source IPv4 ranges**: the CIDR of proxy-psc-subnet + - **Protocols and ports**: Allow all + +### Step 3. Connect from TiDB Cloud + +1. Return to the [TiDB Cloud console](https://tidbcloud.com) and create a changefeed for the cluster to connect to the Kafka cluster by **Private Service Connect**. For more information, see [Sink to Apache Kafka](/tidb-cloud/changefeed-sink-to-apache-kafka.md). + +2. After you proceed to the **Configure the changefeed target** > **Connectivity Method** > **Private Service Connect**, fill in the following fields with corresponding values and other fields as needed. + + - **Kafka Advertised Listener Pattern**: `abc`. The same as the unique random string you use to generate **Kafka Advertised Listener Pattern** in [Prerequisites](#prerequisites). + - **Service Attachment**: the kafka-proxy service attachment of PSC, for example, `projects/tidbcloud-dp-stg-000/regions/us-west1/serviceAttachments/kafka-proxy-psc`. + - **Bootstrap Ports**: `9092,9093,9094` + +3. Continue to follow the guideline in [Sink to Apache Kafka](/tidb-cloud/changefeed-sink-to-apache-kafka.md). + +## FAQ + +### How to connect to the same Kafka Private Service Connect service from two different TiDB Cloud projects? + +If you have already followed the steps in this document and successfully set up the connection from the first project, and you want to set up a second connection from the second project, you can connect to the same Kafka Private Service Connect service from two different TiDB Cloud projects as follows: + +- If you set up Kafka PSC by PSC port mapping, do the following: + + 1. Follow instructions from the beginning of this document. When you proceed to [Step 1. Set up Kafka Cluster](#step-1-set-up-the-kafka-cluster), follow the [Reconfigure a running Kafka cluster](#reconfigure-a-running-kafka-cluster) section to create another group of EXTERNAL listeners and advertised listeners. You can name it as `EXTERNAL2`. Note that the port range of `EXTERNAL2` cannot overlap with the EXTERNAL. + + 2. After reconfiguring the brokers, add another group of Network endpoints to the Network endpoint group, which maps the ports range to the `EXTERNAL2` listener. + + 3. Configure the TiDB Cloud connection with the following input to create the new changefeed: + + - New Bootstrap ports + - New Kafka Advertised Listener Pattern + - The same Service Attachment + +- If you [set up self-hosted Kafka Private Service Connect by Kafka-proxy](#set-up-self-hosted-kafka-private-service-connect-by-kafka-proxy), create a new Kafka-proxy PSC from the beginning with a new Kafka Advertised Listener Pattern. diff --git a/tidb-cloud/size-your-cluster.md b/tidb-cloud/size-your-cluster.md index cd7135fa69170..9824572aa46b5 100644 --- a/tidb-cloud/size-your-cluster.md +++ b/tidb-cloud/size-your-cluster.md @@ -5,11 +5,11 @@ summary: Learn how to determine the size of your TiDB Cloud cluster. # Determine Your TiDB Size -This document describes how to determine the size of a TiDB Dedicated cluster. +This document describes how to determine the size of a TiDB Cloud Dedicated cluster. > **Note:** > -> You cannot change the size of a [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster. +> You cannot change the size of a [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) cluster. ## Size TiDB @@ -23,17 +23,21 @@ To learn performance test results of different cluster scales, see [TiDB Cloud P The supported vCPU and RAM sizes include the following: -- 2 vCPU, 8 GiB (Beta) -- 4 vCPU, 16 GiB -- 8 vCPU, 16 GiB -- 16 vCPU, 32 GiB +| Standard size | High memory size | +|:---------:|:----------------:| +| 4 vCPU, 16 GiB | N/A | +| 8 vCPU, 16 GiB | 8 vCPU, 32 GiB | +| 16 vCPU, 32 GiB | 16 vCPU, 64 GiB | +| 32 vCPU, 64 GiB | 32 vCPU, 128 GiB | > **Note:** > -> If the vCPU and RAM size of TiDB is set as **2 vCPU, 8 GiB (Beta)** or **4 vCPU, 16 GiB**, note the following restrictions: +> To use the **32 vCPU, 128 GiB** size of TiDB, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). +> +> If the vCPU and RAM size of TiDB is set as **4 vCPU, 16 GiB**, note the following restrictions: > > - The node number of TiDB can only be set to 1 or 2, and the node number of TiKV is fixed to 3. -> - 2 vCPU TiDB can only be used with 2 vCPU TiKV. 4 vCPU TiDB can only be used with 4 vCPU TiKV. +> - 4 vCPU TiDB can only be used with 4 vCPU TiKV. > - TiFlash is unavailable. ### TiDB node number @@ -85,18 +89,19 @@ To learn performance test results of different cluster scales, see [TiDB Cloud P The supported vCPU and RAM sizes include the following: -- 2 vCPU, 8 GiB (Beta) -- 4 vCPU, 16 GiB -- 8 vCPU, 32 GiB -- 8 vCPU, 64 GiB -- 16 vCPU, 64 GiB +| Standard size | High memory size | +|:---------:|:----------------:| +| 4 vCPU, 16 GiB | N/A | +| 8 vCPU, 32 GiB | 8 vCPU, 64 GiB | +| 16 vCPU, 64 GiB | Coming soon | +| 32 vCPU, 128 GiB | N/A | > **Note:** > -> If the vCPU and RAM size of TiKV is set as **2 vCPU, 8 GiB (Beta)** or **4 vCPU, 16 GiB**, note the following restrictions: +> If the vCPU and RAM size of TiKV is set as **4 vCPU, 16 GiB**, note the following restrictions: > > - The node number of TiDB can only be set to 1 or 2, and the node number of TiKV is fixed to 3. -> - 2 vCPU TiKV can only be used with 2 vCPU TiDB. 4 vCPU TiKV can only be used with 4 vCPU TiDB. +> - 4 vCPU TiKV can only be used with 4 vCPU TiDB. > - TiFlash is unavailable. ### TiKV node number @@ -168,10 +173,10 @@ The supported node storage of different TiKV vCPUs is as follows: | TiKV vCPU | Min node storage | Max node storage | Default node storage | |:---------:|:----------------:|:----------------:|:--------------------:| -| 2 vCPU | 200 GiB | 500 GiB | 200 GiB | | 4 vCPU | 200 GiB | 2048 GiB | 500 GiB | | 8 vCPU | 200 GiB | 4096 GiB | 500 GiB | | 16 vCPU | 200 GiB | 6144 GiB | 500 GiB | +| 32 vCPU | 200 GiB | 6144 GiB | 500 GiB | > **Note:** > @@ -189,8 +194,10 @@ The supported vCPU and RAM sizes include the following: - 8 vCPU, 64 GiB - 16 vCPU, 128 GiB +- 32 vCPU, 128 GiB +- 32 vCPU, 256 GiB -Note that TiFlash is unavailable when the vCPU and RAM size of TiDB or TiKV is set as **2 vCPU, 8 GiB (Beta)** or **4 vCPU, 16 GiB**. +Note that TiFlash is unavailable when the vCPU and RAM size of TiDB or TiKV is set as **4 vCPU, 16 GiB**. ### TiFlash node number @@ -211,7 +218,8 @@ The supported node storage of different TiFlash vCPUs is as follows: | TiFlash vCPU | Min node storage | Max node storage | Default node storage | |:---------:|:----------------:|:----------------:|:--------------------:| | 8 vCPU | 200 GiB | 2048 GiB | 500 GiB | -| 16 vCPU | 200 GiB | 2048 GiB | 500 GiB | +| 16 vCPU | 200 GiB | 4096 GiB | 500 GiB | +| 32 vCPU | 200 GiB | 4096 GiB | 500 GiB | > **Note:** > diff --git a/tidb-cloud/sql-proxy-account.md b/tidb-cloud/sql-proxy-account.md new file mode 100644 index 0000000000000..6b0d69630b121 --- /dev/null +++ b/tidb-cloud/sql-proxy-account.md @@ -0,0 +1,90 @@ +--- +title: SQL Proxy Account +summary: Learn about the SQL proxy account in TiDB Cloud. +--- + +# SQL Proxy Account + +A SQL proxy account is a SQL user account that is automatically created by TiDB Cloud to access the database via [SQL Editor](/tidb-cloud/explore-data-with-chat2query.md) or [Data Service](https://docs.pingcap.com/tidbcloud/api/v1beta1/dataservice) on behalf of a TiDB Cloud user. For example, `testuser@pingcap.com` is a TiDB Cloud user account, while `3jhEcSimm7keKP8.testuser._41mqK6H4` is its corresponding SQL proxy account. + +SQL proxy accounts provide a secure, token-based authentication mechanism for accessing the database in TiDB Cloud. By eliminating the need for traditional username and password credentials, SQL proxy accounts enhance security and simplify access management. + +The key benefits of SQL proxy accounts are as follows: + +- Enhanced security: mitigates risks associated with static credentials by using JWT tokens. +- Streamlined access: restricts access specifically to the SQL Editor and Data Service, ensuring precise control. +- Ease of management: simplifies authentication for developers and administrators working with TiDB Cloud. + +## Identify the SQL proxy account + +If you want to identify whether a specific SQL account is a SQL proxy account, take the following steps: + +1. Examine the `mysql.user` table: + + ```sql + USE mysql; + SELECT user FROM user WHERE plugin = 'tidb_auth_token'; + ``` + +2. Check grants for the SQL account. If roles like `role_admin`, `role_readonly`, or `role_readwrite` are listed, then it is a SQL proxy account. + + ```sql + SHOW GRANTS for 'username'; + ``` + +## How the SQL proxy account is created + +The SQL proxy account is automatically created during TiDB Cloud cluster initialization for the TiDB Cloud user who is granted a role with permissions in the cluster. + +## How the SQL proxy account is deleted + +When a user is removed from [an organization](/tidb-cloud/manage-user-access.md#remove-an-organization-member) or [a project](/tidb-cloud/manage-user-access.md#remove-a-project-member), or their role changes to one that does not have access to the cluster, the SQL proxy account is automatically deleted. + +Note that if a SQL proxy account is manually deleted, it will be automatically recreated when the user log in to the TiDB Cloud console next time. + +## SQL proxy account username + +In some cases, the SQL proxy account username is exactly the same as the TiDB Cloud username, but in other cases it is not exactly the same. The SQL proxy account username is determined by the length of the TiDB Cloud user's email address. The rules are as follows: + +| Environment | Email length | Username format | +| ----------- | ------------ | --------------- | +| TiDB Cloud Dedicated | <= 32 characters | Full email address | +| TiDB Cloud Dedicated | > 32 characters | `prefix($email, 23)_prefix(base58(sha1($email)), 8)` | +| TiDB Cloud Serverless | <= 15 characters | `serverless_unique_prefix + "." + email` | +| TiDB Cloud Serverless | > 15 characters | `serverless_unique_prefix + "." + prefix($email, 6)_prefix(base58(sha1($email)), 8)` | + +Examples: + +| Environment | Email address | SQL proxy account username | +| ----------- | ----- | -------- | +| TiDB Cloud Dedicated | `user@pingcap.com` | `user@pingcap.com` | +| TiDB Cloud Dedicated | `longemailaddressexample@pingcap.com` | `longemailaddressexample_48k1jwL9` | +| TiDB Cloud Serverless | `u1@pingcap.com` | `{user_name_prefix}.u1@pingcap.com` | +| TiDB Cloud Serverless | `longemailaddressexample@pingcap.com` | `{user_name_prefix}.longem_48k1jwL9`| + +> **Note:** +> +> In the preceding table, `{user_name_prefix}` is a unique prefix generated by TiDB Cloud to distinguish TiDB Cloud Serverless clusters. For details, see the [user name prefix](/tidb-cloud/select-cluster-tier.md#user-name-prefix) of TiDB Cloud Serverless clusters. + +## SQL proxy account password + +Since SQL proxy accounts are JWT token-based, it is not necessary to manage passwords for these accounts. The security token is automatically managed by the system. + +## SQL proxy account roles + +The SQL proxy account's role depends on the TiDB Cloud user's IAM role: + +- Organization level: + - Organization Owner: role_admin + - Organization Billing Admin: No proxy account + - Organization Member: No proxy account + - Organization Console Audit admin: No proxy account + +- Project level: + - Project Owner: role_admin + - Project Data Access Read-Write: role_readwrite + - Project Data Access Read-Only: role_readonly + +## SQL proxy account access control + +SQL proxy accounts are JWT token-based and only accessible to the Data Service and SQL Editor. It is impossible to access the TiDB Cloud cluster using a SQL proxy account with a username and password. diff --git a/tidb-cloud/terraform-get-tidbcloud-provider.md b/tidb-cloud/terraform-get-tidbcloud-provider.md index feebb0ae99d27..1b2637dfc2a00 100644 --- a/tidb-cloud/terraform-get-tidbcloud-provider.md +++ b/tidb-cloud/terraform-get-tidbcloud-provider.md @@ -48,7 +48,7 @@ For detailed steps, see [TiDB Cloud API documentation](https://docs.pingcap.com/ required_providers { tidbcloud = { source = "tidbcloud/tidbcloud" - version = "~> 0.1.0" + version = "~> 0.3.0" } } required_version = ">= 1.0.0" @@ -103,12 +103,28 @@ provider "tidbcloud" { `public_key` and `private_key` are the API key's public key and private key. You can also pass them through the environment variables: ``` -export TIDBCLOUD_PUBLIC_KEY = ${public_key} -export TIDBCLOUD_PRIVATE_KEY = ${private_key} +export TIDBCLOUD_PUBLIC_KEY=${public_key} +export TIDBCLOUD_PRIVATE_KEY=${private_key} ``` Now, you can use the TiDB Cloud Terraform Provider. +## Step 5. Configure TiDB Cloud Terraform Provider with sync configuration + +Terraform provider (>= 0.3.0) supports an optional parameter `sync`. + +By setting `sync` to `true`, you can create, update, or delete resources synchronously. Here is an example: + +``` +provider "tidbcloud" { + public_key = "your_public_key" + private_key = "your_private_key" + sync = true +} +``` + +Setting `sync` to `true` is recommended, but note that `sync` currently only works with the cluster resource. If you need synchronous operations for other resources, [contact TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). + ## Next step -Get started by managing a cluster with the [cluster resource](/tidb-cloud/terraform-use-cluster-resource.md). \ No newline at end of file +Get started by managing a cluster with the [cluster resource](/tidb-cloud/terraform-use-cluster-resource.md). diff --git a/tidb-cloud/terraform-use-backup-resource.md b/tidb-cloud/terraform-use-backup-resource.md index f882f8f2ea20e..e8f843e7853d0 100644 --- a/tidb-cloud/terraform-use-backup-resource.md +++ b/tidb-cloud/terraform-use-backup-resource.md @@ -9,13 +9,13 @@ You can learn how to create a backup of a TiDB Cloud cluster with the `tidbcloud The features of the `tidbcloud_backup` resource include the following: -- Create backups for TiDB Dedicated clusters. -- Delete backups for TiDB Dedicated clusters. +- Create backups for TiDB Cloud Dedicated clusters. +- Delete backups for TiDB Cloud Dedicated clusters. ## Prerequisites - [Get TiDB Cloud Terraform Provider](/tidb-cloud/terraform-get-tidbcloud-provider.md). -- The backup and restore feature is unavailable to TiDB Serverless clusters. To use backup resources, make sure that you have created a TiDB Dedicated cluster. +- The backup and restore feature is unavailable to TiDB Cloud Serverless clusters. To use backup resources, make sure that you have created a TiDB Cloud Dedicated cluster. ## Create a backup with the backup resource diff --git a/tidb-cloud/terraform-use-cluster-resource.md b/tidb-cloud/terraform-use-cluster-resource.md index f297a3a3661ba..66e3b06aae03c 100644 --- a/tidb-cloud/terraform-use-cluster-resource.md +++ b/tidb-cloud/terraform-use-cluster-resource.md @@ -11,9 +11,9 @@ In addition, you will also learn how to get the necessary information with the ` The features of the `tidbcloud_cluster` resource include the following: -- Create TiDB Serverless and TiDB Dedicated clusters. -- Modify TiDB Dedicated clusters. -- Delete TiDB Serverless and TiDB Dedicated clusters. +- Create TiDB Cloud Serverless and TiDB Cloud Dedicated clusters. +- Modify TiDB Cloud Dedicated clusters. +- Delete TiDB Cloud Serverless and TiDB Cloud Dedicated clusters. ## Prerequisites @@ -39,6 +39,7 @@ To view the information of all available projects, you can use the `tidbcloud_pr provider "tidbcloud" { public_key = "your_public_key" private_key = "your_private_key" + sync = true } data "tidbcloud_projects" "example_project" { @@ -137,6 +138,7 @@ To get the cluster specification information, you can use the `tidbcloud_cluster provider "tidbcloud" { public_key = "your_public_key" private_key = "your_private_key" + sync = true } data "tidbcloud_cluster_specs" "example_cluster_spec" { } @@ -158,13 +160,6 @@ To get the cluster specification information, you can use the `tidbcloud_cluster "cluster_type" = "DEDICATED" "region" = "eu-central-1" "tidb" = tolist([ - { - "node_quantity_range" = { - "min" = 1 - "step" = 1 - } - "node_size" = "2C8G" - }, { "node_quantity_range" = { "min" = 1 @@ -212,17 +207,6 @@ To get the cluster specification information, you can use the `tidbcloud_cluster }, ]) "tikv" = tolist([ - { - "node_quantity_range" = { - "min" = 3 - "step" = 3 - } - "node_size" = "2C8G" - "storage_size_gib_range" = { - "max" = 500 - "min" = 200 - } - }, { "node_quantity_range" = { "min" = 3 @@ -285,11 +269,11 @@ In the results: > **Note:** > -> Before you begin, make sure that you have set a Project CIDR in the TiDB Cloud console. For more information, see [Set a Project CIDR](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-project-cidr). +> Before you begin, make sure that you have set a CIDR in the TiDB Cloud console. For more information, see [Set a CIDR](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-cidr-for-a-region). You can create a cluster using the `tidbcloud_cluster` resource. -The following example shows how to create a TiDB Dedicated cluster. +The following example shows how to create a TiDB Cloud Dedicated cluster. 1. Create a directory for the cluster and enter it. @@ -307,6 +291,7 @@ The following example shows how to create a TiDB Dedicated cluster. provider "tidbcloud" { public_key = "your_public_key" private_key = "your_private_key" + sync = true } resource "tidbcloud_cluster" "example_cluster" { @@ -487,9 +472,9 @@ The following example shows how to create a TiDB Dedicated cluster. When the status is `AVAILABLE`, it indicates that your TiDB cluster is created and ready for use. -## Modify a TiDB Dedicated cluster +## Modify a TiDB Cloud Dedicated cluster -For a TiDB Dedicated cluster, you can use Terraform to manage cluster resources as follows: +For a TiDB Cloud Dedicated cluster, you can use Terraform to manage cluster resources as follows: - Add a TiFlash component to the cluster. - Scale the cluster. @@ -622,7 +607,7 @@ You can scale a TiDB cluster when its status is `AVAILABLE`. 1. In the `cluster.tf` file that is used when you [create the cluster](#create-a-cluster-using-the-cluster-resource), edit the `components` configurations. - For example, to add one more node for TiDB, 3 more nodes for TiKV (The number of TiKV nodes needs to be a multiple of 3 for its step is 3. You can [get this information from the cluster specifcation](#get-cluster-specification-information-using-the-tidbcloud_cluster_specs-data-source)), and one more node for TiFlash, you can edit the configurations as follows: + For example, to add one more node for TiDB, 3 more nodes for TiKV (The number of TiKV nodes needs to be a multiple of 3 for its step is 3. You can [get this information from the cluster specification](#get-cluster-specification-information-using-the-tidbcloud_cluster_specs-data-source)), and one more node for TiFlash, you can edit the configurations as follows: ``` components = { @@ -845,7 +830,7 @@ You can pause a cluster when its status is `AVAILABLE` or resume a cluster when 6. Wait for a moment, then use the `terraform refersh` command to update the state. The status will be changed to `AVAILABLE` finally. -Now, you have created and managed a TiDB Dedicated cluster with Terraform. Next, you can try creating a backup of the cluster by our [backup resource](/tidb-cloud/terraform-use-backup-resource.md). +Now, you have created and managed a TiDB Cloud Dedicated cluster with Terraform. Next, you can try creating a backup of the cluster by our [backup resource](/tidb-cloud/terraform-use-backup-resource.md). ## Import a cluster diff --git a/tidb-cloud/terraform-use-import-resource.md b/tidb-cloud/terraform-use-import-resource.md index 333b805ef5c29..3b75f6d3d92d7 100644 --- a/tidb-cloud/terraform-use-import-resource.md +++ b/tidb-cloud/terraform-use-import-resource.md @@ -9,14 +9,14 @@ You can learn how to import data to a TiDB Cloud cluster with the `tidbcloud_imp The features of the `tidbcloud_import` resource include the following: -- Create import tasks for TiDB Serverless and TiDB Dedicated clusters. +- Create import tasks for TiDB Cloud Serverless and TiDB Cloud Dedicated clusters. - Import data either from local disks or from Amazon S3 buckets. - Cancel ongoing import tasks. ## Prerequisites - [Get TiDB Cloud Terraform Provider](/tidb-cloud/terraform-get-tidbcloud-provider.md). -- [Create a TiDB Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) or [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). +- [Create a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) or [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). ## Create and run an import task @@ -26,7 +26,7 @@ You can manage either a local import task or an Amazon S3 import task using the > **Note:** > -> Importing local files is supported only for TiDB Serverless clusters, not for TiDB Dedicated clusters. +> Importing local files is supported only for TiDB Cloud Serverless clusters, not for TiDB Cloud Dedicated clusters. 1. Create a CSV file for import. For example: diff --git a/tidb-cloud/terraform-use-restore-resource.md b/tidb-cloud/terraform-use-restore-resource.md index 9afd88ab46e2f..2793dc7e5ad6b 100644 --- a/tidb-cloud/terraform-use-restore-resource.md +++ b/tidb-cloud/terraform-use-restore-resource.md @@ -9,12 +9,12 @@ You can learn how to manage a restore task with the `tidbcloud_restore` resource The features of the `tidbcloud_restore` resource include the following: -- Create restore tasks for TiDB Dedicated clusters according to your backup. +- Create restore tasks for TiDB Cloud Dedicated clusters according to your backup. ## Prerequisites - [Get TiDB Cloud Terraform Provider](/tidb-cloud/terraform-get-tidbcloud-provider.md). -- The backup and restore feature is unavailable for TiDB Serverless clusters. To use restore resources, make sure that you have created a TiDB Dedicated cluster. +- The backup and restore feature is unavailable for TiDB Cloud Serverless clusters. To use restore resources, make sure that you have created a TiDB Cloud Dedicated cluster. ## Create a restore task diff --git a/tidb-cloud/third-party-monitoring-integrations.md b/tidb-cloud/third-party-monitoring-integrations.md index d0cd59441a9cf..484c6db9b75c7 100644 --- a/tidb-cloud/third-party-monitoring-integrations.md +++ b/tidb-cloud/third-party-monitoring-integrations.md @@ -21,7 +21,7 @@ The available third-party integrations are displayed. ## Limitation -- For [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters, third-party metrics integrations are not supported. +- For [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters, third-party metrics integrations are not supported. - Third-party metrics integrations are not available when the cluster status is **CREATING**, **RESTORING**, **PAUSED**, or **RESUMING**. @@ -35,7 +35,7 @@ For the detailed integration steps and a list of metrics that Datadog tracks, re ### Prometheus and Grafana integration (beta) -With the Prometheus and Grafana integration, you can get a scrape_config file for Prometheus from TiDB Cloud and use the content from the file to configure Prometheus. You can view these metrics in your Grafana dashboards. +With the Prometheus and Grafana integration, you can get a `scrape_config` file for Prometheus from TiDB Cloud and use the content from the file to configure Prometheus. You can view these metrics in your Grafana dashboards. For the detailed integration steps and a list of metrics that Prometheus tracks, see [Integrate TiDB Cloud with Prometheus and Grafana](/tidb-cloud/monitor-prometheus-and-grafana-integration.md). diff --git a/tidb-cloud/ticloud-ai.md b/tidb-cloud/ticloud-ai.md new file mode 100644 index 0000000000000..7f2a6b705c498 --- /dev/null +++ b/tidb-cloud/ticloud-ai.md @@ -0,0 +1,47 @@ +--- +title: ticloud ai +summary: The reference of `ticloud ai`. +--- + +# ticloud ai + +Chat with TiDB Bot: + +```shell +ticloud ai [flags] +``` + +## Examples + +Chat with TiDB Bot in interactive mode: + +```shell +ticloud ai +``` + +Chat with TiDB Bot in non-interactive mode: + +```shell +ticloud ai -q "How to create a cluster?" +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|--------------------|-----------------------------------|----------|------------------------------------------------------| +| -q, --query string | Specifies your query to TiDB Bot. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|--------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-auth-login.md b/tidb-cloud/ticloud-auth-login.md new file mode 100644 index 0000000000000..3ed324f317c67 --- /dev/null +++ b/tidb-cloud/ticloud-auth-login.md @@ -0,0 +1,47 @@ +--- +title: ticloud auth login +summary: The reference of `ticloud auth login`. +--- + +# ticloud auth login + +Authenticate with TiDB Cloud: + +```shell +ticloud auth login [flags] +``` + +## Examples + +To log into TiDB Cloud: + +```shell +ticloud auth login +``` + +To log into TiDB Cloud with insecure storage: + +```shell +ticloud auth login --insecure-storage +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|--------------------|---------------------------------------------------------------------------|----------|------------------------------------------------------| +| --insecure-storage | Saves authentication credentials in plain text instead of credential store. | No | Works in both non-interactive and interactive modes. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|--------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-update.md b/tidb-cloud/ticloud-auth-logout.md similarity index 53% rename from tidb-cloud/ticloud-update.md rename to tidb-cloud/ticloud-auth-logout.md index 0a2c85dd3eca7..39bfa978c3c51 100644 --- a/tidb-cloud/ticloud-update.md +++ b/tidb-cloud/ticloud-auth-logout.md @@ -1,36 +1,31 @@ --- -title: ticloud update -summary: The reference of `ticloud update`. +title: ticloud auth logout +summary: The reference of `ticloud auth logout`. --- -# ticloud update +# ticloud auth logout -Update the TiDB Cloud CLI to the latest version: +Log out of TiDB Cloud: ```shell -ticloud update [flags] +ticloud auth logout [flags] ``` ## Examples -Update the TiDB Cloud CLI to the latest version: +To log out of TiDB Cloud: ```shell -ticloud update +ticloud auth logout ``` -## Flags - -| Flag | Description | -|------------|--------------------------| - | -h, --help | Help information for this command | - ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| +| Flag | Description | Required | Note | +|----------------------|--------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| | --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-auth-whoami.md b/tidb-cloud/ticloud-auth-whoami.md new file mode 100644 index 0000000000000..9d89791916283 --- /dev/null +++ b/tidb-cloud/ticloud-auth-whoami.md @@ -0,0 +1,40 @@ +--- +title: ticloud auth whoami +summary: The reference of `ticloud auth whoami`. +--- + +# ticloud auth whoami + +Display information about the current user: + +```shell +ticloud auth whoami [flags] +``` + +## Examples + +To display information about the current user: + +```shell +ticloud auth whoami +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|------------|------------------------------------------|----------|------------------------------------------------------| +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-branch-connect-info.md b/tidb-cloud/ticloud-branch-connect-info.md deleted file mode 100644 index f2aac9ad26e80..0000000000000 --- a/tidb-cloud/ticloud-branch-connect-info.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: ticloud branch connect-info -summary: The reference of `ticloud branch connect-info`. ---- - -# ticloud branch connect-info - -Get the connection string of a branch: - -```shell -ticloud branch connect-info [flags] -``` - -## Examples - -Get the connection string of a branch in interactive mode: - -```shell -ticloud branch connect-info -``` - -Get the connection string of a branch in non-interactive mode: - -```shell -ticloud branch connect-info --branch-id --cluster-id --client --operating-system -``` - -## Flags - -In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. - -| Flag | Description | Required | Note | -|---------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| -| -c, --cluster-id string | The ID of the cluster, in which the branch is created | Yes | Only works in non-interactive mode. | -| -b, --branch-id string | The ID of the branch | Yes | Only works in non-interactive mode. | -| --client string | The desired client used for the connection. Supported clients include `general`, `mysql_cli`, `mycli`, `libmysqlclient`, `python_mysqlclient`, `pymysql`, `mysql_connector_python`, `mysql_connector_java`, `go_mysql_driver`, `node_mysql2`, `ruby_mysql2`, `php_mysqli`, `rust_mysql`, `mybatis`, `hibernate`, `spring_boot`, `gorm`, `prisma`, `sequelize_mysql2`, `django_tidb`, `sqlalchemy_mysqlclient`, and `active_record`. | Yes | Only works in non-interactive mode. | -| --operating-system string | The operating system name. Supported operating systems include `macOS`, `Windows`, `Ubuntu`, `CentOS`, `RedHat`, `Fedora`, `Debian`, `Arch`, `OpenSUSE`, `Alpine`, and `Others`. | Yes | Only works in non-interactive mode. | -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | - -## Inherited flags - -| Flag | Description | Required | Note | -|----------------------|-------------------------------------------------------------------------------------------------------|----------|-------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | - -## Feedback - -If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-branch-create.md b/tidb-cloud/ticloud-branch-create.md index d73a7dc20853b..5caca96a9f52d 100644 --- a/tidb-cloud/ticloud-branch-create.md +++ b/tidb-cloud/ticloud-branch-create.md @@ -1,50 +1,55 @@ --- -title: ticloud branch create -summary: The reference of `ticloud branch create`. +title: ticloud serverless branch create +summary: The reference of `ticloud serverless branch create`. --- -# ticloud branch create +# ticloud serverless branch create -Create a branch for a cluster: +Create a [branch](/tidb-cloud/branch-overview.md) for a TiDB Cloud Serverless cluster: ```shell -ticloud branch create [flags] +ticloud serverless branch create [flags] ``` -> **Note:** -> -> Currently, you can only create branches for [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster. - ## Examples -Create a branch in interactive mode: +Create a branch for a TiDB Cloud Serverless cluster in interactive mode: + +```shell +ticloud serverless branch create +``` + +Create a branch for a TiDB Cloud Serverless cluster in non-interactive mode: ```shell -ticloud branch create +ticloud serverless branch create --cluster-id --display-name ``` -Create a branch in non-interactive mode: +Create a branch from another branch with a specified timestamp in non-interactive mode: ```shell -ticloud branch create --cluster-id --branch-name +ticloud serverless branch create --cluster-id --display-name --parent-id --parent-timestamp ``` ## Flags In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. -| Flag | Description | Required | Note | -|-------------------------|------------------------------------------------------------|----------|-----------------------------------------------------| -| -c, --cluster-id string | The ID of the cluster, in which the branch will be created | Yes | Only works in non-interactive mode. | -| --branch-name string | The name of the branch to be created | Yes | Only works in non-interactive mode. | -| -h, --help | Get help information for this command | No | Works in both non-interactive and interactive modes | +| Flag | Description | Required | Note | +|---------------------------|-----------------------------------------------------------------------------------------------------------|----------|-----------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster, in which the branch will be created. | Yes | Only works in non-interactive mode. | +| -n, --display-name string | Specifies the name of the branch to be created. | Yes | Only works in non-interactive mode. | +| --parent-id string | Specifies the ID of the branch parent. The default value is the cluster ID. | No | Only works in non-interactive mode. | +| --parent-timestamp string | Specifies the timestamp of the parent branch in RFC3339 format, such as `2024-01-01T00:00:00Z`. The default value is the current time. | No | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|------------------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-branch-delete.md b/tidb-cloud/ticloud-branch-delete.md index 678d7e64bf770..b11209b7f9293 100644 --- a/tidb-cloud/ticloud-branch-delete.md +++ b/tidb-cloud/ticloud-branch-delete.md @@ -1,31 +1,31 @@ --- -title: ticloud branch delete -summary: The reference of `ticloud branch delete`. +title: ticloud serverless branch delete +summary: The reference of `ticloud serverless branch delete`. --- -# ticloud branch delete +# ticloud serverless branch delete -Delete a branch from your cluster: +Delete a branch from your TiDB Cloud Serverless cluster: ```shell -ticloud branch delete [flags] +ticloud serverless branch delete [flags] ``` Or use the following alias command: ```shell -ticloud branch rm [flags] +ticloud serverless branch rm [flags] ``` ## Examples -Delete a branch in interactive mode: +Delete a TiDB Cloud Serverless branch in interactive mode: ```shell -ticloud branch delete +ticloud serverless branch delete ``` -Delete a branch in non-interactive mode: +Delete a TiDB Cloud Serverless branch in non-interactive mode: ```shell ticloud branch delete --branch-id --cluster-id @@ -37,17 +37,18 @@ In non-interactive mode, you need to manually enter the required flags. In inter | Flag | Description | Required | Note | |-------------------------|--------------------------------------------|----------|------------------------------------------------------| -| -b, --branch-id string | The ID of the branch to be deleted | Yes | Only works in non-interactive mode. | -| --force | Deletes a branch without confirmation | No | Works in both non-interactive and interactive modes. | -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| -c, --cluster-id string | The cluster ID of the branch to be deleted | Yes | Only works in non-interactive mode. | +| -b, --branch-id string | Specifies the ID of the branch to be deleted. | Yes | Only works in non-interactive mode. | +| --force | Deletes a branch without confirmation. | No | Works in both non-interactive and interactive modes. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|--------------------------------------------------------------------------------------------|----------|-------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | The active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|--------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-branch-describe.md b/tidb-cloud/ticloud-branch-describe.md index c5d8299a42c86..9074460c63fac 100644 --- a/tidb-cloud/ticloud-branch-describe.md +++ b/tidb-cloud/ticloud-branch-describe.md @@ -1,34 +1,34 @@ --- -title: ticloud branch describe -summary: The reference of `ticloud branch describe`. +title: ticloud serverless branch describe +summary: The reference of `ticloud serverless branch describe`. --- -# ticloud branch describe +# ticloud serverless branch describe Get information about a branch (such as the endpoints, [user name prefix](/tidb-cloud/select-cluster-tier.md#user-name-prefix), and usages): ```shell -ticloud branch describe [flags] +ticloud serverless branch describe [flags] ``` Or use the following alias command: ```shell -ticloud branch get [flags] +ticloud serverless branch get [flags] ``` ## Examples -Get the branch information in interactive mode: +Get branch information of a TiDB Cloud Serverless cluster in interactive mode: ```shell -ticloud branch describe +ticloud serverless branch describe ``` -Get the branch information in non-interactive mode: +Get branch information of a TiDB Cloud Serverless cluster in non-interactive mode: ```shell -ticloud branch describe --branch-id --cluster-id +ticloud serverless branch describe --branch-id --cluster-id ``` ## Flags @@ -37,16 +37,17 @@ In non-interactive mode, you need to manually enter the required flags. In inter | Flag | Description | Required | Note | |-------------------------|-----------------------------------|----------|------------------------------------------------------| -| -b, --branch-id string | The ID of the branch | Yes | Only works in non-interactive mode. | -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| -c, --cluster-id string | The cluster ID of the branch | Yes | Only works in non-interactive mode. | +| -b, --branch-id string | Specifies the ID of the branch. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|------------------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-branch-list.md b/tidb-cloud/ticloud-branch-list.md index a3cacf5d665d5..805ffa5a7d574 100644 --- a/tidb-cloud/ticloud-branch-list.md +++ b/tidb-cloud/ticloud-branch-list.md @@ -1,65 +1,59 @@ --- -title: ticloud branch list -summary: The reference of `ticloud branch list`. +title: ticloud serverless branch list +summary: The reference of `ticloud serverless branch list`. --- -# ticloud branch list +# ticloud serverless branch list -List all branches for a cluster: +List all branches for a TiDB Cloud Serverless cluster: ```shell -ticloud branch list [flags] +ticloud serverless branch list [flags] ``` Or use the following alias command: ```shell -ticloud branch ls [flags] +ticloud serverless branch ls [flags] ``` ## Examples -List all branches for a cluster (interactive mode): +List all branches for a TiDB Cloud Serverless cluster in interactive mode: ```shell -ticloud branch list +ticloud serverless branch list ``` -List all branches for a specified cluster (non-interactive mode): +List all branches for a specific TiDB Cloud Serverless cluster in non-interactive mode: ```shell -ticloud branch list +ticloud serverless branch list -c ``` -List all branches for a specified cluster in the JSON format: +List all branches for a specific TiDB Cloud Serverless cluster in the JSON format: ```shell -ticloud branch list -o json +ticloud serverless branch list -o json ``` -## Arguments - -The `branch list` command has the following arguments: - -| Argument Index | Description | Required | Note | -|----------------|-----------------------------------------------------|----------|---------------------------------------| -| `` | The cluster ID of the branches which will be listed | Yes | Only works in non-interactive mode. | - ## Flags In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. -| Flag | Description | Required | Note | -|---------------------|--------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| -o, --output string | Output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|-------------------------|--------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| -o, --output string | Specifies the output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|------------------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-branch-shell.md b/tidb-cloud/ticloud-branch-shell.md new file mode 100644 index 0000000000000..835cd99f440fe --- /dev/null +++ b/tidb-cloud/ticloud-branch-shell.md @@ -0,0 +1,63 @@ +--- +title: ticloud serverless branch shell +summary: The reference of `ticloud serverless branch shell`. +aliases: ['/tidbcloud/ticloud-connect'] +--- + +# ticloud serverless branch shell + +Connect to a branch of a TiDB Cloud Serverless cluster: + +```shell +ticloud serverless branch shell [flags] +``` + +## Examples + +Connect to a TiDB Cloud Serverless branch in interactive mode: + +```shell +ticloud serverless branch shell +``` + +Connect to a TiDB Cloud Serverless branch with the default user in non-interactive mode: + +```shell +ticloud serverless branch shell -c -b +``` + +Connect to a TiDB Cloud Serverless branch with the default user and password in non-interactive mode: + +```shell +ticloud serverless branch shell -c -b --password +``` + +Connect to a TiDB Cloud Serverless branch with a specific user and password in non-interactive mode: + +```shell +ticloud serverless branch shell -c -b -u --password +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|-----------------------------------|----------|------------------------------------------------------| +| -b, --branch-id string | Specifies the ID of the branch. | Yes | Only works in non-interactive mode. | +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| --password | Specifies the password of the user. | No | Only works in non-interactive mode. | +| -u, --user string | Specifies the user for login. | No | Only works in non-interactive mode. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. \ No newline at end of file diff --git a/tidb-cloud/ticloud-cluster-connect-info.md b/tidb-cloud/ticloud-cluster-connect-info.md deleted file mode 100644 index 0e2c4ec1e28e1..0000000000000 --- a/tidb-cloud/ticloud-cluster-connect-info.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -title: ticloud cluster connect-info -summary: The reference of `ticloud cluster connect-info`. ---- - -# ticloud cluster connect-info - -Get the connection string of a cluster: - -```shell -ticloud cluster connect-info [flags] -``` - -> **Note:** -> -> Currently, this command only supports getting the connection string of a [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster. - -## Examples - -Get the connection string of a cluster in interactive mode: - -```shell -ticloud cluster connect-info -``` - -Get the connection string of a cluster in non-interactive mode: - -```shell -ticloud cluster connect-info --project-id --cluster-id --client --operating-system -``` - -## Flags - -In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. - -| Flag | Description | Required | Note | -|----------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| -| -p, --project-id string | The ID of the project, in which the cluster is created | Yes | Only works in non-interactive mode. | -| -c, --cluster-id string | The ID of the cluster | Yes | Only works in non-interactive mode. | -| --client string | The desired client used for the connection. Supported clients include `general`, `mysql_cli`, `mycli`, `libmysqlclient`, `python_mysqlclient`, `pymysql`, `mysql_connector_python`, `mysql_connector_java`, `go_mysql_driver`, `node_mysql2`, `ruby_mysql2`, `php_mysqli`, `rust_mysql`, `mybatis`, `hibernate`, `spring_boot`, `gorm`, `prisma`, `sequelize_mysql2`, `django_tidb`, `sqlalchemy_mysqlclient`, and `active_record`. | Yes | Only works in non-interactive mode. | -| --operating-system string | The operating system name. Supported operating systems include `macOS`, `Windows`, `Ubuntu`, `CentOS`, `RedHat`, `Fedora`, `Debian`, `Arch`, `OpenSUSE`, `Alpine`, and `Others`. | Yes | Only works in non-interactive mode. | -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | - -## Inherited flags - -| Flag | Description | Required | Note | -|----------------------|-------------------------------------------------------------------------------------------------------|----------|-------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | - -## Feedback - -If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-cluster-create.md b/tidb-cloud/ticloud-cluster-create.md index f9dbd9ba85b10..d295836efe330 100644 --- a/tidb-cloud/ticloud-cluster-create.md +++ b/tidb-cloud/ticloud-cluster-create.md @@ -1,54 +1,57 @@ --- -title: ticloud cluster create -summary: The reference of `ticloud cluster create`. +title: ticloud serverless create +summary: The reference of `ticloud serverless create`. --- -# ticloud cluster create +# ticloud serverless create -Create a cluster: +Create a TiDB Cloud Serverless cluster: ```shell -ticloud cluster create [flags] +ticloud serverless create [flags] ``` -> **Note:** -> -> Currently, you can only create a [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster using the preceding command. - ## Examples -Create a cluster in interactive mode: +Create a TiDB Cloud Serverless cluster in interactive mode: ```shell -ticloud cluster create +ticloud serverless create ``` -Create a cluster in non-interactive mode: +Create a TiDB Cloud Serverless cluster in non-interactive mode: ```shell -ticloud cluster create --project-id --cluster-name --cloud-provider --region --root-password --cluster-type +ticloud serverless create --display-name --region ``` +Create a TiDB Cloud Serverless cluster with a spending limit in non-interactive mode: + +```shell +ticloud serverless create --display-name --region --spending-limit-monthly +``` + ## Flags In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. -| Flag | Description | Required | Note | -|-------------------------|-------------------------------------------------------------|----------|-----------------------------------| -| --cloud-provider string | Cloud provider (Currently, only `AWS` is supported) | Yes | Only works in non-interactive mode. | -| --cluster-name string | Name of the cluster to be created | Yes | Only works in non-interactive mode. | -| --cluster-type string | Cluster type (Currently, only `SERVERLESS` is supported) | Yes | Only works in non-interactive mode. | -| -h, --help | Get help information for this command | No | Works in both non-interactive and interactive modes | -| -p, --project-id string | The ID of the project, in which the cluster will be created | Yes | Only works in non-interactive mode. | -| -r, --region string | Cloud region | Yes | Only works in non-interactive mode. | -| --root-password string | The root password of the cluster | Yes | Only works in non-interactive mode. | +| Flag | Description | Required | Note | +|------------------------------|----------------------------------------------------------------------------------------------------------------|----------|-----------------------------------------------------| +| -n --display-name string | Specifies the name of the cluster to be created. | Yes | Only works in non-interactive mode. | +| --spending-limit-monthly int | Specifies the maximum monthly spending limit in USD cents. | No | Only works in non-interactive mode. | +| -p, --project-id string | Specifies the ID of the project, in which the cluster will be created. The default value is `default project`. | No | Only works in non-interactive mode. | +| -r, --region string | Specifies the name of cloud region. You can use "ticloud serverless region" to see all regions. | Yes | Only works in non-interactive mode. | +| --disable-public-endpoint | Disables the public endpoint. | No | Only works in non-interactive mode. | +| --encryption | Enables enhanced encryption at rest. | No | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-cluster-delete.md b/tidb-cloud/ticloud-cluster-delete.md index 080e92df19a11..3bbb3342cdd77 100644 --- a/tidb-cloud/ticloud-cluster-delete.md +++ b/tidb-cloud/ticloud-cluster-delete.md @@ -1,53 +1,53 @@ --- -title: ticloud cluster delete -summary: The reference of `ticloud cluster delete`. +title: ticloud serverless cluster delete +summary: The reference of `ticloud serverless delete`. --- -# ticloud cluster delete +# ticloud serverless delete -Delete a cluster from your project: +Delete a TiDB Cloud Serverless cluster from your project: ```shell -ticloud cluster delete [flags] +ticloud serverless delete [flags] ``` Or use the following alias command: ```shell -ticloud cluster rm [flags] +ticloud serverless rm [flags] ``` ## Examples -Delete a cluster in interactive mode: +Delete a TiDB Cloud Serverless cluster in interactive mode: ```shell -ticloud cluster delete +ticloud serverless delete ``` -Delete a cluster in non-interactive mode: +Delete a TiDB Cloud Serverless cluster in non-interactive mode: ```shell -ticloud cluster delete --project-id --cluster-id +ticloud serverless delete --cluster-id ``` ## Flags In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. -| Flag | Description | Required | Note | -|-------------------------|---------------------------------------------|----------|-----------------------------------------------------| -| -c, --cluster-id string | The ID of the cluster to be deleted | Yes | Only works in non-interactive mode. | -| --force | Deletes a cluster without confirmation | No | Works in both non-interactive and interactive modes. | -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| -p, --project-id string | The project ID of the cluster to be deleted | Yes | Only works in non-interactive mode. | +| Flag | Description | Required | Note | +|-------------------------|----------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster to be deleted. | Yes | Only works in non-interactive mode. | +| --force | Deletes a cluster without confirmation. | No | Works in both non-interactive and interactive modes. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| +| Flag | Description | Required | Note | +|----------------------|--------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| | --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | The active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-cluster-describe.md b/tidb-cloud/ticloud-cluster-describe.md index 2215ecccb1264..7f23d0efdba06 100644 --- a/tidb-cloud/ticloud-cluster-describe.md +++ b/tidb-cloud/ticloud-cluster-describe.md @@ -1,52 +1,52 @@ --- -title: ticloud cluster describe -summary: The reference of `ticloud cluster describe`. +title: ticloud serverless cluster describe +summary: The reference of `ticloud serverless describe`. --- -# ticloud cluster describe +# ticloud serverless describe -Get information about a cluster (such as the cloud provider, cluster type, cluster configurations, and cluster status): +Get information about a TiDB Cloud Serverless cluster (such as the cluster configurations and cluster status): ```shell -ticloud cluster describe [flags] +ticloud serverless describe [flags] ``` Or use the following alias command: ```shell -ticloud cluster get [flags] +ticloud serverless get [flags] ``` ## Examples -Get the cluster information in interactive mode: +Get information about a TiDB Cloud Serverless cluster in interactive mode: ```shell -ticloud cluster describe +ticloud serverless describe ``` -Get the cluster information in non-interactive mode: +Get information about a TiDB Cloud Serverless cluster in non-interactive mode: ```shell -ticloud cluster describe --project-id --cluster-id +ticloud serverless describe --cluster-id ``` ## Flags In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. -| Flag | Description | Required | Note | -|-------------------------|-------------------------------|----------|-----------------------------------| -| -c, --cluster-id string | The ID of the cluster | Yes | Only works in non-interactive mode. | -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| -p, --project-id string | The project ID of the cluster | Yes | Only works in non-interactive mode. | +| Flag | Description | Required | Note | +|-------------------------|-----------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-cluster-list.md b/tidb-cloud/ticloud-cluster-list.md index a8d05cea36a75..50092432c309c 100644 --- a/tidb-cloud/ticloud-cluster-list.md +++ b/tidb-cloud/ticloud-cluster-list.md @@ -1,57 +1,59 @@ --- -title: ticloud cluster list -summary: The reference of `ticloud cluster list`. +title: ticloud serverless cluster list +summary: The reference of `ticloud serverless list`. --- -# ticloud cluster list +# ticloud serverless list -List all clusters in a project: +List all TiDB Cloud Serverless clusters in a project: ```shell -ticloud cluster list [flags] +ticloud serverless list [flags] ``` Or use the following alias command: ```shell -ticloud cluster ls [flags] +ticloud serverless ls [flags] ``` ## Examples -List all clusters in a project (interactive mode): +List all TiDB Cloud Serverless clusters in interactive mode: ```shell -ticloud cluster list +ticloud serverless list ``` -List all clusters in a specified project (non-interactive mode): +List all TiDB Cloud Serverless clusters in a specified project in non-interactive mode: ```shell -ticloud cluster list +ticloud serverless list -p ``` -List all clusters in a specified project in the JSON format: +List all TiDB Cloud Serverless clusters in a specified project with the JSON format in non-interactive mode: ```shell -ticloud cluster list -o json +ticloud serverless list -p -o json ``` ## Flags In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. -| Flag | Description | Required | Note | -|---------------------|--------------------------------------------------------------------------------------------------------|----------|-----------------------------------------------------| -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| -o, --output string | Output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|-------------------------|--------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| +| -p, --project-id string | Specifies the ID of the project. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| -o, --output string | Specifies the output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-completion.md b/tidb-cloud/ticloud-completion.md new file mode 100644 index 0000000000000..2c563df51d004 --- /dev/null +++ b/tidb-cloud/ticloud-completion.md @@ -0,0 +1,46 @@ +--- +title: ticloud completion +summary: The reference of `ticloud completion`. +--- + +# ticloud completion + +Generate the autocompletion script for the specified shell of TiDB Cloud CLI: + +```shell +ticloud completion [command] +``` + +## Examples + +Generate the autocompletion script for bash: + +```shell +ticloud completion bash +``` + +Generate the autocompletion script for zsh: + +```shell +ticloud completion zsh +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|---------------------------------------------------------------|----------|------------------------------------------------------| +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. \ No newline at end of file diff --git a/tidb-cloud/ticloud-config-create.md b/tidb-cloud/ticloud-config-create.md index c5d2f566984ce..74d7a64720323 100644 --- a/tidb-cloud/ticloud-config-create.md +++ b/tidb-cloud/ticloud-config-create.md @@ -35,10 +35,10 @@ In non-interactive mode, you need to manually enter the required flags. In inter | Flag | Description | Required | Note | |-----------------------|-----------------------------------------------|----------|-----------------------------------| -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| --private-key string | The private key of the TiDB Cloud API | Yes | Only works in non-interactive mode. | -| --profile-name string | The name of the profile, which must not contain `.` | Yes | Only works in non-interactive mode. | -| --public-key string | The public key of the TiDB Cloud API | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| --private-key string | Specifies the private key of the TiDB Cloud API. | Yes | Only works in non-interactive mode. | +| --profile-name string | Specifies the name of the profile (which must not contain `.`). | Yes | Only works in non-interactive mode. | +| --public-key string | Specifies the public key of the TiDB Cloud API. | Yes | Only works in non-interactive mode. | ## Inherited flags @@ -46,6 +46,7 @@ In non-interactive mode, you need to manually enter the required flags. In inter |----------------------|----------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| | --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | | -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-config-delete.md b/tidb-cloud/ticloud-config-delete.md index 6e039e49c0075..e9e35fb5759ba 100644 --- a/tidb-cloud/ticloud-config-delete.md +++ b/tidb-cloud/ticloud-config-delete.md @@ -29,8 +29,8 @@ ticloud config delete | Flag | Description | |------------|---------------------------------------| -| --force | Deletes a profile without confirmation | -| -h, --help | Help information for this command | +| --force | Deletes a profile without confirmation. | +| -h, --help | Shows help information for this command. | ## Inherited flags @@ -38,6 +38,7 @@ ticloud config delete |----------------------|-----------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| | --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | | -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-config-describe.md b/tidb-cloud/ticloud-config-describe.md index acdf34e102a6b..802421ce671a9 100644 --- a/tidb-cloud/ticloud-config-describe.md +++ b/tidb-cloud/ticloud-config-describe.md @@ -29,7 +29,7 @@ ticloud config describe | Flag | Description | |------------|--------------------------| -| -h, --help | Help information for this command | +| -h, --help | Shows help information for this command. | ## Inherited flags @@ -37,6 +37,7 @@ ticloud config describe |----------------------|-----------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| | --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | | -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-config-edit.md b/tidb-cloud/ticloud-config-edit.md index 6fb82659297b8..83389b8c148db 100644 --- a/tidb-cloud/ticloud-config-edit.md +++ b/tidb-cloud/ticloud-config-edit.md @@ -29,7 +29,7 @@ ticloud config edit | Flag | Description | |------------|--------------------------| - | -h, --help | Help information for this command | + | -h, --help | Shows help information for this command. | ## Inherited flags @@ -37,6 +37,7 @@ ticloud config edit |----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| | --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | | -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-config-list.md b/tidb-cloud/ticloud-config-list.md index 766f7c8e99f2d..549005ed9fcea 100644 --- a/tidb-cloud/ticloud-config-list.md +++ b/tidb-cloud/ticloud-config-list.md @@ -29,7 +29,7 @@ ticloud config list | Flag | Description | |------------|--------------------------| -| -h, --help | Help information for this command | +| -h, --help | Shows help information for this command. | ## Inherited flags @@ -37,6 +37,7 @@ ticloud config list |----------------------|-----------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| | --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | | -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-config-set.md b/tidb-cloud/ticloud-config-set.md index 8059c79e9fcc7..bd8d5f8227221 100644 --- a/tidb-cloud/ticloud-config-set.md +++ b/tidb-cloud/ticloud-config-set.md @@ -15,11 +15,11 @@ The properties that can be configured include `public-key`, `private-key`, and ` | Properties | Description | Required | |-------------|--------------------------------------------------------------------|----------| -| public-key | The public key of the TiDB Cloud API | Yes | -| private-key | The private key of the TiDB Cloud API | Yes | -| api-url | The base API URL of TiDB Cloud (`https://api.tidbcloud.com` by default) | No | +| public-key | Specifies the public key of the TiDB Cloud API. | Yes | +| private-key | Specifies the private key of the TiDB Cloud API. | Yes | +| api-url | Specifies the base API URL of TiDB Cloud (`https://api.tidbcloud.com` by default). | No | -> **Notes:** +> **Note:** > > If you want to configure properties for a specific user profile, you can add the `-P` flag and specify the target user profile name in the command. @@ -51,7 +51,7 @@ ticloud config set api-url https://api.tidbcloud.com | Flag | Description | |------------|--------------------------| -| -h, --help | Help information for this command | +| -h, --help | Shows help information for this command. | ## Inherited flags @@ -59,6 +59,7 @@ ticloud config set api-url https://api.tidbcloud.com |----------------------|-----------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| | --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | | -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-config-use.md b/tidb-cloud/ticloud-config-use.md index d73b9c76860f1..197735abface4 100644 --- a/tidb-cloud/ticloud-config-use.md +++ b/tidb-cloud/ticloud-config-use.md @@ -23,7 +23,7 @@ ticloud config use test | Flag | Description | |------------|--------------------------| -| -h, --help | Help information for this command | +| -h, --help | Shows help information for this command. | ## Inherited flags @@ -31,6 +31,7 @@ ticloud config use test |----------------------|-----------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| | --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | | -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-connect.md b/tidb-cloud/ticloud-connect.md deleted file mode 100644 index 93a631f3f7eb7..0000000000000 --- a/tidb-cloud/ticloud-connect.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: ticloud connect -summary: The reference of `ticloud connect`. ---- - -# ticloud connect - -Connect to a TiDB Cloud cluster or branch: - -```shell -ticloud connect [flags] -``` - -> **Note:** -> -> - If you are prompted about whether to use the default user, you can choose `Y` to use the default root user or choose `n` to specify another user. For [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters, the name of the default root user has a [prefix](/tidb-cloud/select-cluster-tier.md#user-name-prefix) such as `3pTAoNNegb47Uc8`. -> - The connection forces the [ANSI SQL mode](https://dev.mysql.com/doc/refman/8.0/en/sql-mode.html#sqlmode_ansi) for the session. To exit the session, enter `\q`. - -## Examples - -Connect to a TiDB Cloud cluster or branch in interactive mode: - -```shell -ticloud connect -``` - -Use the default user to connect to a TiDB Cloud cluster or branch in non-interactive mode: - -```shell -ticloud connect -p -c -ticloud connect -p -c -b -``` - -Use the default user to connect to the TiDB Cloud cluster or branch with password in non-interactive mode: - -```shell -ticloud connect -p -c --password -ticloud connect -p -c -b --password -``` - -Use a specific user to connect to the TiDB Cloud cluster or branch in non-interactive mode: - -```shell -ticloud connect -p -c -u -ticloud connect -p -c -b -u -``` - -## Flags - -In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. - -| Flag | Description | Required | Note | -|-------------------------|-----------------------------------|----------|------------------------------------------------------| -| -c, --cluster-id string | Cluster ID | Yes | Only works in non-interactive mode. | -| -b, --branch-id string | Branch ID | No | Only works in non-interactive mode. | -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| --password | The password of the user | No | Only works in non-interactive mode. | -| -p, --project-id string | Project ID | Yes | Only works in non-interactive mode. | -| -u, --user string | A specific user for login | No | Only works in non-interactive mode. | - -## Inherited flags - -| Flag | Description | Required | Note | -|----------------------|------------------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | - -## Feedback - -If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-help.md b/tidb-cloud/ticloud-help.md new file mode 100644 index 0000000000000..11aa4732d3e6d --- /dev/null +++ b/tidb-cloud/ticloud-help.md @@ -0,0 +1,46 @@ +--- +title: ticloud help +summary: The reference of `ticloud help`. +--- + +# ticloud help + +Get help information for any command in TiDB Cloud CLI: + +```shell +ticloud help [command] [flags] +``` + +## Examples + +To get help for the `auth` command: + +```shell +ticloud help auth +``` + +To get help for the `serverless create` command: + +```shell +ticloud help serverless create +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|---------------------------------------------------------------|----------|------------------------------------------------------| +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. \ No newline at end of file diff --git a/tidb-cloud/ticloud-import-cancel.md b/tidb-cloud/ticloud-import-cancel.md index bb42e358db4c7..8ce763008620d 100644 --- a/tidb-cloud/ticloud-import-cancel.md +++ b/tidb-cloud/ticloud-import-cancel.md @@ -1,14 +1,14 @@ --- -title: ticloud import cancel -summary: The reference of `ticloud import cancel`. +title: ticloud serverless import cancel +summary: The reference of `ticloud serverless import cancel`. --- -# ticloud import cancel +# ticloud serverless import cancel Cancel a data import task: ```shell -ticloud import cancel [flags] +ticloud serverless import cancel [flags] ``` ## Examples @@ -16,26 +16,25 @@ ticloud import cancel [flags] Cancel an import task in interactive mode: ```shell -ticloud import cancel +ticloud serverless import cancel ``` Cancel an import task in non-interactive mode: ```shell -ticloud import cancel --project-id --cluster-id --import-id +ticloud serverless import cancel --cluster-id --import-id ``` ## Flags In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. -| Flag | Description | Required | Note | -|-------------------------|---------------------------------------|----------|-----------------------------------------------------| -| -c, --cluster-id string | Cluster ID | Yes | Only works in non-interactive mode. | -| --force | Deletes a profile without confirmation | No | Works in both non-interactive and interactive modes. | -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| --import-id string | The ID of the import task | Yes | Only works in non-interactive mode. | -| -p, --project-id string | Project ID | Yes | Only works in non-interactive mode. | +| Flag | Description | Required | Note | +|-------------------------|----------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| --force | Cancels an import task without confirmation. | No | Works in both non-interactive and interactive modes. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| --import-id string | Specifies the ID of the import task. | Yes | Only works in non-interactive mode. | ## Inherited flags @@ -43,6 +42,7 @@ In non-interactive mode, you need to manually enter the required flags. In inter |----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| | --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | | -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-import-describe.md b/tidb-cloud/ticloud-import-describe.md index c01062e039c2f..bcbf64fcdb025 100644 --- a/tidb-cloud/ticloud-import-describe.md +++ b/tidb-cloud/ticloud-import-describe.md @@ -1,20 +1,20 @@ --- -title: ticloud import describe -summary: The reference of `ticloud import describe`. +title: ticloud serverless import describe +summary: The reference of `ticloud serverless import describe`. --- -# ticloud import describe +# ticloud serverless import describe -Get the import details of a data import task: +Describe a data import task: ```shell -ticloud import describe [flags] +ticloud serverless import describe [flags] ``` Or use the following alias command: ```shell -ticloud import get [flags] +ticloud serverless import get [flags] ``` ## Examples @@ -22,32 +22,32 @@ ticloud import get [flags] Describe an import task in interactive mode: ```shell -ticloud import describe +ticloud serverless import describe ``` Describe an import task in non-interactive mode: ```shell -ticloud import describe --project-id --cluster-id --import-id +ticloud serverless import describe --cluster-id --import-id ``` ## Flags In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. -| Flag | Description | Required | Note | -|-------------------------|--------------------------|----------|-----------------------------------| -| -c, --cluster-id string | Cluster ID | Yes | Only works in non-interactive mode. | -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| --import-id string | The ID of the import task | Yes | Only works in non-interactive mode. | -| -p, --project-id string | Project ID | Yes | Only works in non-interactive mode. | +| Flag | Description | Required | Note | +|-------------------------|-----------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| --import-id string | Specifies the ID of the import task. | Yes | Only works in non-interactive mode. | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-import-list.md b/tidb-cloud/ticloud-import-list.md index d02d5c26b6798..55cd733c718d8 100644 --- a/tidb-cloud/ticloud-import-list.md +++ b/tidb-cloud/ticloud-import-list.md @@ -1,20 +1,20 @@ --- -title: ticloud import list -summary: The reference of `ticloud import list`. +title: ticloud serverless import list +summary: The reference of `ticloud serverless import list`. --- -# ticloud import list +# ticloud serverless import list List data import tasks: ```shell -ticloud import list [flags] +ticloud serverless import list [flags] ``` Or use the following alias command: ```shell -ticloud import ls [flags] +ticloud serverless import ls [flags] ``` ## Examples @@ -22,38 +22,38 @@ ticloud import ls [flags] List import tasks in interactive mode: ```shell -ticloud import list +ticloud serverless import list ``` List import tasks in non-interactive mode: ```shell -ticloud import list --project-id --cluster-id +ticloud serverless import list --cluster-id ``` List import tasks for a specified cluster in the JSON format: ```shell -ticloud import list --project-id --cluster-id --output json +ticloud serverless import list --cluster-id --output json ``` ## Flags In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. -| Flag | Description | Required | Note | -|-------------------------|--------------------------------------------------------------------------------------------------------|----------|-----------------------------------------------------| -| -c, --cluster-id string | Cluster ID | Yes | Only works in non-interactive mode. | -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| -o, --output string | Output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | -| -p, --project-id string | Project ID | Yes | Only works in non-interactive mode. | +| Flag | Description | Required | Note | +|-------------------------|--------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| -o, --output string | Specifies the output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-import-start-local.md b/tidb-cloud/ticloud-import-start-local.md deleted file mode 100644 index c3576a4f901ae..0000000000000 --- a/tidb-cloud/ticloud-import-start-local.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -title: ticloud import start local -summary: The reference of `ticloud import start local`. ---- - -# ticloud import start local - -Import a local file to a [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster: - -```shell -ticloud import start local [flags] -``` - -> **Note:** -> -> Currently, you can only import one CSV file for one import task. - -## Examples - -Start an import task in interactive mode: - -```shell -ticloud import start local -``` - -Start an import task in non-interactive mode: - -```shell -ticloud import start local --project-id --cluster-id --data-format --target-database --target-table -``` - -Start an import task with a custom CSV format: - -```shell -ticloud import start local --project-id --cluster-id --data-format CSV --target-database --target-table --separator \" --delimiter \' --backslash-escape=false --trim-last-separator=true -``` - -## Flags - -In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. - -| Flag | Description | Required | Note | -|---|---|---|---| -| --backslash-escape | Whether to parse backslashes inside fields as escape characters for CSV files. The default value is `true`. | No | Only works in non-interactive mode when `--data-format CSV` is specified. | -| -c, --cluster-id string | Specifies the cluster ID. | Yes | Only works in non-interactive mode. | -| --data-format string | Specifies the data format. Currently, only `CSV` is supported. | Yes | Only works in non-interactive mode. | -| --delimiter string | Specifies the delimiter used for quoting for CSV files. The default value is `"`. | No | Only works in non-interactive mode when `--data-format CSV` is specified. | -| -h, --help | Displays help information for this command. | No | Works in both non-interactive and interactive modes. | -| -p, --project-id string | Specifies the project ID. | Yes | Only works in non-interactive mode. | -| --separator string | Specifies the field separator for CSV files. The default value is `,`. | No | Only works in non-interactive mode when `--data-format CSV` is specified. | -| --target-database string | Specifies the target database to import data to. | Yes | Only works in non-interactive mode. | -| --target-table string | Specifies the target table to import data to. | Yes | Only works in non-interactive mode. | -| --trim-last-separator | Whether to treat separators as line terminators and trim all trailing separators for CSV files. The default value is `false`. | No | Only works in non-interactive mode when `--data-format CSV` is specified. | - -## Inherited flags - -| Flag | Description | Required | Note | -|---|---|---|---| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | - -## Feedback - -If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-import-start-mysql.md b/tidb-cloud/ticloud-import-start-mysql.md deleted file mode 100644 index 3c67cc007d95a..0000000000000 --- a/tidb-cloud/ticloud-import-start-mysql.md +++ /dev/null @@ -1,79 +0,0 @@ ---- -title: ticloud import start mysql -summary: The reference of `ticloud import start mysql`. ---- - -# ticloud import start mysql - -Import a table from a MySQL-compatible database to a [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster: - -```shell -ticloud import start mysql [flags] -``` - -> **Note:** -> -> - Before running this command, make sure that you have installed the MySQL command-line tool first. For more details, see [Installation](/tidb-cloud/get-started-with-cli.md#installation). -> - If the target table already exists in the target database, to use this command for table import, make sure that the target table name is the same as the source table name and add the `skip-create-table` flag in the command. -> - If the target table does not exist in the target database, executing this command automatically creates a table with the same name as the source table in the target database. - -## Examples - -- Start an import task in interactive mode: - - ```shell - ticloud import start mysql - ``` - -- Start an import task in non-interactive mode (using the TiDB Serverless cluster default user `.root`): - - ```shell - ticloud import start mysql --project-id --cluster-id --source-host --source-port --source-user --source-password --source-database --source-table --target-database --target-password - ``` - -- Start an import task in non-interactive mode (using a specific user): - - ```shell - ticloud import start mysql --project-id --cluster-id --source-host --source-port --source-user --source-password --source-database --source-table --target-database --target-password --target-user - ``` - -- Start an import task that skips creating the target table if it already exists in the target database: - - ```shell - ticloud import start mysql --project-id --cluster-id --source-host --source-port --source-user --source-password --source-database --source-table --target-database --target-password --skip-create-table - ``` - -> **Note:** -> -> MySQL 8.0 uses `utf8mb4_0900_ai_ci` as the default collation, which is currently not supported by TiDB. If your source table uses the `utf8mb4_0900_ai_ci` collation, before the import, you need to either alter the source table collation to a [supported collation of TiDB](/character-set-and-collation.md#character-sets-and-collations-supported-by-tidb) or manually create the target table in TiDB. - -## Flags - -In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. - -| Flag | Description | Required | Note | -|---|---|---|---| -| -c, --cluster-id string | Specifies the cluster ID. | Yes | Only works in non-interactive mode. | -| -h, --help | Displays help information for this command. | No | Works in both non-interactive and interactive modes. | -| -p, --project-id string | Specifies the project ID. | Yes | Only works in non-interactive mode. | -| --skip-create-table | Skips creating the target table if it already exists in the target database. | No | Only works in non-interactive mode. | -| --source-database string | The name of the source MySQL database. | Yes | Only works in non-interactive mode. | -| --source-host string | The host of the source MySQL instance. | Yes | Only works in non-interactive mode. | -| --source-password string | The password for the source MySQL instance. | Yes | Only works in non-interactive mode. | -| --source-port int | The port of the source MySQL instance. | Yes | Only works in non-interactive mode. | -| --source-table string | The source table name in the source MySQL database. | Yes | Only works in non-interactive mode. | -| --source-user string | The user to log in to the source MySQL instance. | Yes | Only works in non-interactive mode. | -| --target-database string | The target database name in the TiDB Serverless cluster. | Yes | Only works in non-interactive mode. | -| --target-password string | The password for the target TiDB Serverless cluster. | Yes | Only works in non-interactive mode. | -| --target-user string | The user to log in to the target TiDB Serverless cluster. | No | Only works in non-interactive mode. | - -## Inherited flags - -| Flag | Description | Required | Note | -|---|---|---|---| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | - -## Feedback - -If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-import-start-s3.md b/tidb-cloud/ticloud-import-start-s3.md deleted file mode 100644 index d42f84def8128..0000000000000 --- a/tidb-cloud/ticloud-import-start-s3.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -title: ticloud import start s3 -summary: The reference of `ticloud import start s3`. ---- - -# ticloud import start s3 - -Import files from Amazon S3 into TiDB Cloud: - -```shell -ticloud import start s3 [flags] -``` - -> **Note:** -> -> Before importing files from Amazon S3 into TiDB Cloud, you need to configure the Amazon S3 bucket access for TiDB Cloud and get the Role ARN. For more information, see [Configure Amazon S3 access](/tidb-cloud/config-s3-and-gcs-access.md#configure-amazon-s3-access). - -## Examples - -Start an import task in interactive mode: - -```shell -ticloud import start s3 -``` - -Start an import task in non-interactive mode: - -```shell -ticloud import start s3 --project-id --cluster-id --aws-role-arn --data-format --source-url -``` - -Start an import task with a custom CSV format: - -```shell -ticloud import start s3 --project-id --cluster-id --aws-role-arn --data-format CSV --source-url --separator \" --delimiter \' --backslash-escape=false --trim-last-separator=true -``` - -## Flags - -In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. - -| Flag | Description | Required | Note | -|---|---|---|---| -| --aws-role-arn string | Specifies the AWS role ARN that is used to access the Amazon S3 data source. | Yes | Only works in non-interactive mode. | -| --backslash-escape | Whether to parse backslashes inside fields as escape characters for CSV files. The default value is `true`. | No | Only works in non-interactive mode when `--data-format CSV` is specified. | -| -c, --cluster-id string | Specifies the cluster ID. | Yes | Only works in non-interactive mode. | -| --data-format string | Specifies the data format. Valid values are `CSV`, `SqlFile`, `Parquet`, or `AuroraSnapshot`. | Yes | Only works in non-interactive mode. | -| --delimiter string | Specifies the delimiter used for quoting for CSV files. The default value is `"`. | No | Only works in non-interactive mode when `--data-format CSV` is specified. | -| -h, --help | Displays help information for this command. | No | Works in both non-interactive and interactive modes. | -| -p, --project-id string | Specifies the project ID. | Yes | Only works in non-interactive mode. | -| --separator string | Specifies the field separator for CSV files. The default value is `,`. | No | Only works in non-interactive mode when `--data-format CSV` is specified. | -| --source-url string | The S3 path where the source data files are stored. | Yes | Only works in non-interactive mode. | -| --trim-last-separator | Whether to treat separators as line terminators and trim all trailing separators for CSV files. The default value is `false`. | No | Only works in non-interactive mode when `--data-format CSV` is specified. | - -## Inherited flags - -| Flag | Description | Required | Note | -|---|---|---|---| -| --no-color | Disables color in output | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | - -## Feedback - -If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-import-start.md b/tidb-cloud/ticloud-import-start.md new file mode 100644 index 0000000000000..778baf3283b01 --- /dev/null +++ b/tidb-cloud/ticloud-import-start.md @@ -0,0 +1,109 @@ +--- +title: ticloud serverless import start +summary: The reference of `ticloud serverless import start`. +aliases: ['/tidbcloud/ticloud-import-start-local','/tidbcloud/ticloud-import-start-mysql','/tidbcloud/ticloud-import-start-s3'] +--- + +# ticloud serverless import start + +Start a data import task: + +```shell +ticloud serverless import start [flags] +``` + +Or use the following alias command: + +```shell +ticloud serverless import create [flags] +``` + +> **Note:** +> +> Currently, you can only import one CSV file for one local import task. + +## Examples + +Start an import task in interactive mode: + +```shell +ticloud serverless import start +``` + +Start a local import task in non-interactive mode: + +```shell +ticloud serverless import start --local.file-path --cluster-id --file-type --local.target-database --local.target-table +``` + +Start a local import task with custom upload concurrency: + +```shell +ticloud serverless import start --local.file-path --cluster-id --file-type --local.target-database --local.target-table --local.concurrency 10 +``` + +Start a local import task with custom CSV format: + +```shell +ticloud serverless import start --local.file-path --cluster-id --file-type CSV --local.target-database --local.target-table --csv.separator \" --csv.delimiter \' --csv.backslash-escape=false --csv.trim-last-separator=true +``` + +Start an S3 import task in non-interactive mode: + +```shell +ticloud serverless import start --source-type S3 --s3.uri --cluster-id --file-type --s3.role-arn +``` + +Start a GCS import task in non-interactive mode: + +```shell +ticloud serverless import start --source-type GCS --gcs.uri --cluster-id --file-type --gcs.service-account-key +``` + +Start an Azure Blob import task in non-interactive mode: + +```shell +ticloud serverless import start --source-type AZURE_BLOB --azblob.uri --cluster-id --file-type --azblob.sas-token +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|----------------------------------|----------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| +| --azblob.sas-token string | Specifies the SAS token of Azure Blob. | No | Only works in non-interactive mode. | +| --azblob.uri string | Specifies the Azure Blob URI in `azure://.blob.core.windows.net//` format. | No | Only works in non-interactive mode. | +| --gcs.service-account-key string | Specifies the base64 encoded service account key of GCS. | No | Only works in non-interactive mode. | +| --gcs.uri string | Specifies the GCS URI in `gcs:///` format. Required when source type is GCS. | Yes | Only works in non-interactive mode. | +| --s3.access-key-id string | Specifies the access key ID of Amazon S3. You only need to set one of the `s3.role-arn` and [`s3.access-key-id`, `s3.secret-access-key`]. | No | Only works in non-interactive mode. | +| --s3.role-arn string | Specifies the role ARN of Amazon S3. You only need to set one of the `s3.role-arn` and [`s3.access-key-id`, `s3.secret-access-key`]. | No | Only works in non-interactive mode. | +| --s3.secret-access-key string | Specifies the secret access key of Amazon S3. You only need to set one of the `s3.role-arn` and [`s3.access-key-id`, `s3.secret-access-key`]. | No | Only works in non-interactive mode. | +| --s3.uri string | Specifies the S3 URI in `s3:///` format. Required when source type is S3. | Yes | Only works in non-interactive mode. | +| --source-type string | Specifies the import source type, one of [`"LOCAL"` `"S3"` `"GCS"` `"AZURE_BLOB"`]. The default value is `"LOCAL"`. | No | Only works in non-interactive mode. | +| -c, --cluster-id string | Specifies the cluster ID. | Yes | Only works in non-interactive mode. | +| --local.concurrency int | Specifies the concurrency for uploading files. The default value is `5`. | No | Only works in non-interactive mode. | +| --local.file-path string | Specifies the path of the local file to be imported. | No | Only works in non-interactive mode. | +| --local.target-database string | Specifies the target database to which the data is imported. | No | Only works in non-interactive mode. | +| --local.target-table string | Specifies the target table to which the data is imported. | No | Only works in non-interactive mode. | +| --file-type string | Specifies the import file type, one of ["CSV" "SQL" "AURORA_SNAPSHOT" "PARQUET"]. | Yes | Only works in non-interactive mode. | +| --csv.backslash-escape | Specifies whether to parse backslash inside fields as escape characters in a CSV file. The default value is `true`. | No | Only works in non-interactive mode. | +| --csv.delimiter string | Specifies the delimiter used for quoting a CSV file. The default value is `\`. | No | Only works in non-interactive mode. | +| --csv.separator string | Specifies the field separator in a CSV file. The default value is `,`. | No | Only works in non-interactive mode. | +| --csv.skip-header | Specifies whether the CSV file contains a header line. | No | Only works in non-interactive mode. | +| --csv.trim-last-separator | Specifies whether to treat the separator as the line terminator and trim all trailing separators in a CSV file. | No | Only works in non-interactive mode. | +| --csv.not-null | Specifies whether a CSV file can contain any NULL values. | No | Only works in non-interactive mode. | +| --csv.null-value string | Specifies the representation of NULL values in the CSV file. (default "\\N") | No | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-project-list.md b/tidb-cloud/ticloud-project-list.md index eee14b32cd698..da1fba930bcd9 100644 --- a/tidb-cloud/ticloud-project-list.md +++ b/tidb-cloud/ticloud-project-list.md @@ -35,17 +35,18 @@ ticloud project list -o json In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. -| Flag | Description | Required | Note | -|---------------------|--------------------------------------------------------------------------------------------------------|----------|-----------------------------------------------------| -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| -o, --output string | Output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|---------------------|--------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| -o, --output string | Specifies the output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-serverless-export-cancel.md b/tidb-cloud/ticloud-serverless-export-cancel.md new file mode 100644 index 0000000000000..777ec6686b79a --- /dev/null +++ b/tidb-cloud/ticloud-serverless-export-cancel.md @@ -0,0 +1,49 @@ +--- +title: ticloud serverless export cancel +summary: The reference of `ticloud serverless export cancel`. +--- + +# ticloud serverless export cancel + +Cancel a data export task: + +```shell +ticloud serverless export cancel [flags] +``` + +## Examples + +Cancel an export task in interactive mode: + +```shell +ticloud serverless export cancel +``` + +Cancel an export task in non-interactive mode: + +```shell +ticloud serverless export cancel -c -e +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|----------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -e, --export-id string | Specifies the ID of the export task. | Yes | Only works in non-interactive mode. | +| --force | Cancels an export task without confirmation | No | Works in both non-interactive and interactive modes. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-export-create.md b/tidb-cloud/ticloud-serverless-export-create.md new file mode 100644 index 0000000000000..7caf468707bd5 --- /dev/null +++ b/tidb-cloud/ticloud-serverless-export-create.md @@ -0,0 +1,97 @@ +--- +title: ticloud serverless export create +summary: The reference of `ticloud serverless export create`. +--- + +# ticloud serverless export create + +Export data from a TiDB Cloud Serverless cluster: + +```shell +ticloud serverless export create [flags] +``` + +## Examples + +Export data from a TiDB Cloud Serverless cluster in interactive mode: + +```shell +ticloud serverless export create +``` + +Export data from a TiDB Cloud Serverless cluster to a local file in non-interactive mode: + +```shell +ticloud serverless export create -c --filter +``` + +Export data from a TiDB Cloud Serverless cluster to Amazon S3 in non-interactive mode: + +```shell +ticloud serverless export create -c --s3.uri --s3.access-key-id --s3.secret-access-key --filter +``` + +Export data from a TiDB Cloud Serverless cluster to Google Cloud Storage in non-interactive mode: + +```shell +ticloud serverless export create -c --gcs.uri --gcs.service-account-key --filter +``` + +Export data from a TiDB Cloud Serverless cluster to Azure Blob Storage in non-interactive mode: + +```shell +ticloud serverless export create -c --azblob.uri --azblob.sas-token --filter +``` + +Export data to a Parquet file and compress it with `SNAPPY` in non-interactive mode: + +```shell +ticloud serverless export create -c --file-type parquet --parquet.compression SNAPPY --filter +``` + +Export data with SQL statements in non-interactive mode: + +```shell +ticloud serverless export create -c --sql 'select * from database.table' +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|----------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster, from which you want to export data. | Yes | Only works in non-interactive mode. | +| --file-type string | Specifies the export file type. One of ["SQL" "CSV" "PARQUET"]. (default "CSV") | No | Only works in non-interactive mode. | +| --target-type string | Specifies the export target. One of [`"LOCAL"` `"S3"` `"GCS"` `"AZURE_BLOB"`]. The default value is `"LOCAL"`. | No | Only works in non-interactive mode. | +| --s3.uri string | Specifies the S3 URI in `s3:///` format. Required when the target type is S3. | No | Only works in non-interactive mode. | +| --s3.access-key-id string | Specifies the access key ID of Amazon S3. You only need to set one of the s3.role-arn and [s3.access-key-id, s3.secret-access-key]. | NO | Only works in non-interactive mode. | +| --s3.secret-access-key string | Specifies the secret access key of Amazon S3. You only need to set one of the s3.role-arn and [s3.access-key-id, s3.secret-access-key]. | No | Only works in non-interactive mode. | +| --s3.role-arn string | Specifies the role ARN of Amazon S3. You only need to set one of the s3.role-arn and [s3.access-key-id, s3.secret-access-key]. | No | Only works in non-interactive mode. | +| --gcs.uri string | Specifies the GCS URI in `gcs:///` format. Required when the target type is GCS. | No | Only works in non-interactive mode. | +| --gcs.service-account-key string | Specifies the base64 encoded service account key of GCS. | No | Only works in non-interactive mode. | +| --azblob.uri string | Specifies the Azure Blob URI in `azure://.blob.core.windows.net//` format. Required when the target type is AZURE_BLOB. | No | Only works in non-interactive mode. | +| --azblob.sas-token string | Specifies the SAS token of Azure Blob. | No | Only works in non-interactive mode. | +| --csv.delimiter string | Specifies the delimiter of string type variables in CSV files. (default "\"") | No | Only works in non-interactive mode. | +| --csv.null-value string | Specifies the representation of null values in CSV files. (default "\\N") | No | Only works in non-interactive mode. | +| --csv.separator string | Specifies the separator of each value in CSV files. (default ",") | No | Only works in non-interactive mode. | +| --csv.skip-header | Exports CSV files of the tables without header. | No | Only works in non-interactive mode. | +| --parquet.compression string | Specifies the Parquet compression algorithm. One of [`"GZIP"` `"SNAPPY"` `"ZSTD"` `"NONE"`]. The default value is `"ZSTD"`. | No | Only works in non-interactive mode. | +| --filter strings | Specifies the exported tables with table filter patterns. Do not use it with --sql. For more information, see [Table Filter](/table-filter.md). | No | Only works in non-interactive mode. | +| --sql string | Filters the exported data with the `SQL SELECT` statement. | No | Only works in non-interactive mode. | +| --where string | Filters the exported tables with the `WHERE` condition. Do not use it with --sql. | No | Only works in non-interactive mode. | +| --compression string | Specifies the compression algorithm of the export file. The supported algorithms include `GZIP`, `SNAPPY`, `ZSTD`, and `NONE`. The default value is `GZIP`. | No | Only works in non-interactive mode. | +| --force | Creates the export task without confirmation. You need to confirm when you want to export the whole cluster in non-interactive mode. | No | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-export-describe.md b/tidb-cloud/ticloud-serverless-export-describe.md new file mode 100644 index 0000000000000..ea9fd4ca7311c --- /dev/null +++ b/tidb-cloud/ticloud-serverless-export-describe.md @@ -0,0 +1,54 @@ +--- +title: ticloud serverless export describe +summary: The reference of `ticloud serverless export describe`. +--- + +# ticloud serverless export describe + +Get the export information of a TiDB Cloud Serverless cluster: + +```shell +ticloud serverless export describe [flags] +``` + +Or use the following alias command: + +```shell +ticloud serverless export get [flags] +``` + +## Examples + +Get the export information in interactive mode: + +```shell +ticloud serverless export describe +``` + +Get the export information in non-interactive mode: + +```shell +ticloud serverless export describe -c -e +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|----------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -e, --export-id string | Specifies the ID of the export task. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-export-download.md b/tidb-cloud/ticloud-serverless-export-download.md new file mode 100644 index 0000000000000..bb1ee9b9dc5de --- /dev/null +++ b/tidb-cloud/ticloud-serverless-export-download.md @@ -0,0 +1,51 @@ +--- +title: ticloud serverless export download +summary: The reference of `ticloud serverless export download`. +--- + +# ticloud serverless export download + +Download the exported data from a TiDB Cloud Serverless cluster to your local storage: + +```shell +ticloud serverless export download [flags] +``` + +## Examples + +Download the exported data in interactive mode: + +```shell +ticloud serverless export download +``` + +Download the exported data in non-interactive mode: + +```shell +ticloud serverless export download -c -e +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|-----------------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -e, --export-id string | Specifies the ID of the export task. | Yes | Only works in non-interactive mode. | +| --output-path string | Specifies the destination path for saving the downloaded data. If not specified, the data is downloaded to the current directory. | No | Only works in non-interactive mode. | +| --concurrency int | Specifies the download concurrency. The default value is `3`. | No | Works in both non-interactive and interactive modes. | +| --force | Downloads the exported data without confirmation. | No | Works in both non-interactive and interactive modes. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-export-list.md b/tidb-cloud/ticloud-serverless-export-list.md new file mode 100644 index 0000000000000..03a16afc5884f --- /dev/null +++ b/tidb-cloud/ticloud-serverless-export-list.md @@ -0,0 +1,60 @@ +--- +title: ticloud serverless export list +summary: The reference of `ticloud serverless export list`. +--- + +# ticloud serverless export list + +List data export tasks of TiDB Cloud Serverless clusters: + +```shell +ticloud serverless export list [flags] +``` + +Or use the following alias command: + +```shell +ticloud serverless export ls [flags] +``` + +## Examples + +List all export tasks in interactive mode: + +```shell +ticloud serverless export list +``` + +List export tasks for a specified cluster in non-interactive mode: + +```shell +ticloud serverless export list -c +``` + +List export tasks for a specified cluster in the JSON format in non-interactive mode: + +```shell +ticloud serverless export list -c -o json +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|--------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -o, --output string | Specifies the output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-region.md b/tidb-cloud/ticloud-serverless-region.md new file mode 100644 index 0000000000000..6e78e5d87f524 --- /dev/null +++ b/tidb-cloud/ticloud-serverless-region.md @@ -0,0 +1,48 @@ +--- +title: ticloud serverless region +summary: The reference of `ticloud serverless region`. +aliases: ['/tidbcloud/ticloud-serverless-regions'] +--- + +# ticloud serverless region + +List all available regions for TiDB Cloud Serverless: + +```shell +ticloud serverless region [flags] +``` + +## Examples + +List all available regions for TiDB Cloud Serverless: + +```shell +ticloud serverless region +``` + +List all available regions for TiDB Cloud Serverless clusters in the JSON format: + +```shell +ticloud serverless region -o json +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|---------------------|--------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| +| -o, --output string | Specifies the output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|--------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-shell.md b/tidb-cloud/ticloud-serverless-shell.md new file mode 100644 index 0000000000000..9ae3d7eec5dd3 --- /dev/null +++ b/tidb-cloud/ticloud-serverless-shell.md @@ -0,0 +1,62 @@ +--- +title: ticloud serverless shell +summary: The reference of `ticloud serverless shell`. +aliases: ['/tidbcloud/ticloud-connect'] +--- + +# ticloud serverless shell + +Connect to a TiDB Cloud Serverless cluster: + +```shell +ticloud serverless shell [flags] +``` + +## Examples + +Connect to a TiDB Cloud Serverless cluster in interactive mode: + +```shell +ticloud serverless shell +``` + +Connect to a TiDB Cloud Serverless cluster with the default user in non-interactive mode: + +```shell +ticloud serverless shell -c +``` + +Connect to a TiDB Cloud Serverless cluster with the default user and password in non-interactive mode: + +```shell +ticloud serverless shell -c --password +``` + +Connect to a TiDB Cloud Serverless cluster with a specific user and password in non-interactive mode: + +```shell +ticloud connect -c -u --password +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|-----------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| --password | Specifies the password of the user. | No | Only works in non-interactive mode. | +| -u, --user string | Specifies the user for login. | No | Only works in non-interactive mode. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. \ No newline at end of file diff --git a/tidb-cloud/ticloud-serverless-spending-limit.md b/tidb-cloud/ticloud-serverless-spending-limit.md new file mode 100644 index 0000000000000..5abca8a06666d --- /dev/null +++ b/tidb-cloud/ticloud-serverless-spending-limit.md @@ -0,0 +1,48 @@ +--- +title: ticloud serverless spending-limit +summary: The reference of `ticloud serverless spending-limit`. +--- + +# ticloud serverless spending-limit + +Set the maximum monthly [spending limit](/tidb-cloud/manage-serverless-spend-limit.md) for a TiDB Cloud Serverless cluster: + +```shell +ticloud serverless spending-limit [flags] +``` + +## Examples + +Set the spending limit for a TiDB Cloud Serverless cluster in interactive mode: + +```shell +ticloud serverless spending-limit +``` + +Set the spending limit for a TiDB Cloud Serverless cluster in non-interactive mode: + +```shell +ticloud serverless spending-limit -c --monthly +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|---------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| --monthly int32 | Specifies the maximum monthly spending limit in USD cents. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|--------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-sql-user-create.md b/tidb-cloud/ticloud-serverless-sql-user-create.md new file mode 100644 index 0000000000000..894360c4ffdda --- /dev/null +++ b/tidb-cloud/ticloud-serverless-sql-user-create.md @@ -0,0 +1,50 @@ +--- +title: ticloud serverless sql-user create +summary: The reference of `ticloud serverless sql-user create`. +--- + +# ticloud serverless sql-user create + +Create a TiDB Cloud Serverless SQL user: + +```shell +ticloud serverless sql-user create [flags] +``` + +## Examples + +Create a TiDB Cloud Serverless SQL user in interactive mode: + +```shell +ticloud serverless sql-user create +``` + +Create a TiDB Cloud Serverless SQL user in non-interactive mode: + +```shell +ticloud serverless sql-user create --user --password --role --cluster-id +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| --password string | Specifies the password of the SQL user. | No | Only works in non-interactive mode. | +| --role strings | Specifies the roles of the SQL user. | No | Only works in non-interactive mode. | +| -u, --user string | Specifies the name of the SQL user. | No | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-sql-user-delete.md b/tidb-cloud/ticloud-serverless-sql-user-delete.md new file mode 100644 index 0000000000000..9f7cd35ea52d9 --- /dev/null +++ b/tidb-cloud/ticloud-serverless-sql-user-delete.md @@ -0,0 +1,48 @@ +--- +title: ticloud serverless sql-user delete +summary: The reference of `ticloud serverless sql-user delete`. +--- + +# ticloud serverless sql-user delete + +Delete a TiDB Cloud Serverless SQL user: + +```shell +ticloud serverless sql-user delete [flags] +``` + +## Examples + +Delete a TiDB Cloud Serverless SQL user in interactive mode: + +```shell +ticloud serverless sql-user delete +``` + +Delete a TiDB Cloud Serverless SQL user in non-interactive mode: + +```shell +ticloud serverless sql-user delete -c --user +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| --force | Deletes a SQL user without confirmation. | No | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-sql-user-list.md b/tidb-cloud/ticloud-serverless-sql-user-list.md new file mode 100644 index 0000000000000..152b0362e0701 --- /dev/null +++ b/tidb-cloud/ticloud-serverless-sql-user-list.md @@ -0,0 +1,48 @@ +--- +title: ticloud serverless sql-user list +summary: The reference of `ticloud serverless sql-user list`. +--- + +# ticloud serverless sql-user list + +List TiDB Cloud Serverless SQL users: + +```shell +ticloud serverless sql-user list [flags] +``` + +## Examples + +List TiDB Cloud Serverless SQL users in interactive mode: + +```shell +ticloud serverless sql-user list +``` + +List TiDB Cloud Serverless SQL users in non-interactive mode: + +```shell +ticloud serverless sql-user list -c +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|-------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -o, --output string | Specifies the output format, one of ["human" "json"]. To get a complete result, use "json" format. (default "human"). | No | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-sql-user-update.md b/tidb-cloud/ticloud-serverless-sql-user-update.md new file mode 100644 index 0000000000000..1925b3280fc57 --- /dev/null +++ b/tidb-cloud/ticloud-serverless-sql-user-update.md @@ -0,0 +1,52 @@ +--- +title: ticloud serverless sql-user update +summary: The reference of `ticloud serverless sql-user update`. +--- + +# ticloud serverless sql-user update + +Update a TiDB Cloud Serverless SQL user: + +```shell +ticloud serverless sql-user update [flags] +``` + +## Examples + +Update a TiDB Cloud Serverless SQL user in interactive mode: + +```shell +ticloud serverless sql-user update +``` + +Update a TiDB Cloud Serverless SQL user in non-interactive mode: + +```shell +ticloud serverless sql-user update -c --user --password --role +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|-------------------------------------------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| --password string | Specifies the new password of the SQL user. | No | Only works in non-interactive mode. | +| --role strings | Specifies the new roles of the SQL user. Passing this flag replaces existing roles. | No | Only works in non-interactive mode. | +| --add-role strings | Specifies the roles to be added to the SQL user. | No | Only works in non-interactive mode. | +| --delete-role strings | Specifies the roles to be deleted from the SQL user. | No | Only works in non-interactive mode. | +| -u, --user string | Specifies the name of the SQL user to be updated. | No | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-update.md b/tidb-cloud/ticloud-serverless-update.md new file mode 100644 index 0000000000000..a8c00ce24feb1 --- /dev/null +++ b/tidb-cloud/ticloud-serverless-update.md @@ -0,0 +1,56 @@ +--- +title: ticloud serverless update +summary: The reference of `ticloud serverless update`. +--- + +# ticloud serverless update + +Update a TiDB Cloud Serverless cluster: + +```shell +ticloud serverless update [flags] +``` + +## Examples + +Update a TiDB Cloud Serverless cluster in interactive mode: + +```shell +ticloud serverless update +``` + +Update the name of a TiDB Cloud Serverless cluster in non-interactive mode: + +```shell +ticloud serverless update -c --display-name +``` + +Update labels of a TiDB Cloud Serverless cluster in non-interactive mode + +```shell +ticloud serverless update -c --labels "{\"label1\":\"value1\"}" +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|---------------------------|---------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -n --display-name string | Specifies a new name for the cluster. | No | Only works in non-interactive mode. |. +| --labels string | Specifies new labels for the cluster. | No | Only works in non-interactive mode. | +| --disable-public-endpoint | Disables the public endpoint of the cluster. | No | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-upgrade.md b/tidb-cloud/ticloud-upgrade.md new file mode 100644 index 0000000000000..ad033b6063338 --- /dev/null +++ b/tidb-cloud/ticloud-upgrade.md @@ -0,0 +1,39 @@ +--- +title: ticloud upgrade +summary: The reference of `ticloud upgrade`. +aliases: ['/tidbcloud/ticloud-update'] +--- + +# ticloud upgrade + +Upgrade the TiDB Cloud CLI to the latest version: + +```shell +ticloud upgrade [flags] +``` + +## Examples + +Upgrade the TiDB Cloud CLI to the latest version: + +```shell +ticloud upgrade +``` + +## Flags + +| Flag | Description | +|------------|-----------------------------------| + | -h, --help | Shows help information for this command. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/tidb-cloud-auditing.md b/tidb-cloud/tidb-cloud-auditing.md index 85215aa23ba01..1e3c11065ed8f 100644 --- a/tidb-cloud/tidb-cloud-auditing.md +++ b/tidb-cloud/tidb-cloud-auditing.md @@ -9,7 +9,7 @@ TiDB Cloud provides you with a database audit logging feature to record a histor > **Note:** > -> Currently, the database audit logging feature is only available upon request. To request this feature, click **?** in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com) and click **Request Support**. Then, fill in "Apply for database audit logging" in the **Description** field and click **Send**. +> Currently, the database audit logging feature is only available upon request. To request this feature, click **?** in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com) and click **Request Support**. Then, fill in "Apply for database audit logging" in the **Description** field and click **Submit**. To assess the effectiveness of user access policies and other information security measures of your organization, it is a security best practice to conduct a periodic analysis of the database audit logs. @@ -21,12 +21,12 @@ The audit logging feature is disabled by default. To audit a cluster, you need t ## Prerequisites -- You are using a TiDB Dedicated cluster. Audit logging is not available for TiDB Serverless clusters. +- You are using a TiDB Cloud Dedicated cluster. Audit logging is not available for TiDB Cloud Serverless clusters. - You are in the `Organization Owner` or `Project Owner` role of your organization. Otherwise, you cannot see the database audit-related options in the TiDB Cloud console. For more information, see [User roles](/tidb-cloud/manage-user-access.md#user-roles). -## Enable audit logging for AWS or Google Cloud +## Enable audit logging -To allow TiDB Cloud to write audit logs to your cloud bucket, you need to enable audit logging first. +TiDB Cloud supports recording the audit logs of a TiDB Cloud Dedicated cluster to your cloud storage service. Before enabling database audit logging, configure your cloud storage service on the cloud provider where the cluster is located. ### Enable audit logging for AWS @@ -40,16 +40,17 @@ For more information, see [Creating a bucket](https://docs.aws.amazon.com/Amazon #### Step 2. Configure Amazon S3 access -> **Note:** -> -> Once the Amazon S3 access configuration is performed for one cluster in a project, you can use the same bucket as a destination for audit logs from all clusters in the same project. +1. Get the TiDB Cloud Account ID and the External ID of the TiDB cluster that you want to enable audit logging. + + 1. In the TiDB Cloud console, navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. -1. Get the TiDB Cloud account ID and the External ID of the TiDB cluster that you want to enable audit logging. + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. - 1. In the TiDB Cloud console, choose a project and a cluster deployed on AWS. - 2. Select **Settings** > **Audit Settings**. The **Audit Logging** dialog is displayed. - 3. In the **Audit Logging** dialog, click **Show AWS IAM policy settings**. The corresponding TiDB Cloud Account ID and TiDB Cloud External ID of the TiDB cluster are displayed. - 4. Record the TiDB Cloud Account ID and the External ID for later use. + 2. Click the name of your target cluster to go to its overview page, and then click **DB Audit Logging** in the left navigation pane. + 3. On the **DB Audit Logging** page, click **Enable** in the upper-right corner. + 4. In the **Enable Database Audit Logging** dialog, locate the **AWS IAM Policy Settings** section, and record **TiDB Cloud Account ID** and **TiDB Cloud External ID** for later use. 2. In the AWS Management Console, go to **IAM** > **Access Management** > **Policies**, and then check whether there is a storage bucket policy with the `s3:PutObject` write-only permission. @@ -83,23 +84,23 @@ For more information, see [Creating a bucket](https://docs.aws.amazon.com/Amazon #### Step 3. Enable audit logging -In the TiDB Cloud console, go back to the **Audit Logging** dialog box where you got the TiDB Cloud account ID and the External ID values, and then take the following steps: +In the TiDB Cloud console, go back to the **Enable Database Audit Logging** dialog box where you got the TiDB Cloud account ID and the External ID values, and then take the following steps: 1. In the **Bucket URI** field, enter the URI of your S3 bucket where the audit log files are to be written. 2. In the **Bucket Region** drop-down list, select the AWS region where the bucket locates. 3. In the **Role ARN** field, fill in the Role ARN value that you copied in [Step 2. Configure Amazon S3 access](#step-2-configure-amazon-s3-access). -4. Click **Test Connectivity** to verify whether TiDB Cloud can access and write to the bucket. +4. Click **Test Connection** to verify whether TiDB Cloud can access and write to the bucket. - If it is successful, **Pass** is displayed. Otherwise, check your access configuration. + If it is successful, **The connection is successfully** is displayed. Otherwise, check your access configuration. -5. In the upper-right corner, toggle the audit setting to **On**. +5. Click **Enable** to enable audit logging for the cluster. TiDB Cloud is ready to write audit logs for the specified cluster to your Amazon S3 bucket. > **Note:** > -> - After enabling audit logging, if you make any new changes to the bucket URI, location, or ARN, you must click **Restart** to load the changes and rerun the **Test Connectivity** check to make the changes effective. -> - To remove Amazon S3 access from TiDB Cloud, simply delete the trust policy that you added. +> - After enabling audit logging, if you make any new changes to the bucket URI, location, or ARN, you must click **Test Connection** again to verify that TiDB Cloud can connect to the bucket. Then, click **Enable** to apply the changes. +> - To remove TiDB Cloud's access to your Amazon S3, simply delete the trust policy granted to this cluster in the AWS Management Console. ### Enable audit logging for Google Cloud @@ -113,15 +114,17 @@ For more information, see [Creating storage buckets](https://cloud.google.com/st #### Step 2. Configure GCS access -> **Note:** -> -> Once the GCS access configuration is performed for one cluster in a project, you can use the same bucket as a destination for audit logs from all clusters in the same project. - 1. Get the Google Cloud Service Account ID of the TiDB cluster that you want to enable audit logging. - 1. In the TiDB Cloud console, choose a project and a cluster deployed on Google Cloud Platform. - 2. Select **Settings** > **Audit Settings**. The **Audit Logging** dialog box is displayed. - 3. Click **Show Google Cloud Service Account ID**, and then copy the Service Account ID for later use. + 1. In the TiDB Cloud console, navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + + 2. Click the name of your target cluster to go to its overview page, and then click **DB Audit Logging** in the left navigation pane. + 3. On the **DB Audit Logging** page, click **Enable** in the upper-right corner. + 4. In the **Enable Database Audit Logging** dialog, locate the **Google Cloud Server Account ID** section, and record **Service Account ID** for later use. 2. In the Google Cloud console, go to **IAM & Admin** > **Roles**, and then check whether a role with the following write-only permissions of the storage container exists. @@ -146,22 +149,22 @@ For more information, see [Creating storage buckets](https://cloud.google.com/st #### Step 3. Enable audit logging -In the TiDB Cloud console, go back to the **Audit Logging** dialog box where you got the TiDB Cloud account ID, and then take the following steps: +In the TiDB Cloud console, go back to the **Enable Database Audit Logging** dialog box where you got the TiDB Cloud account ID, and then take the following steps: 1. In the **Bucket URI** field, enter your full GCS bucket name. 2. In the **Bucket Region** field, select the GCS region where the bucket locates. -3. Click **Test Connectivity** to verify whether TiDB Cloud can access and write to the bucket. +3. Click **Test Connection** to verify whether TiDB Cloud can access and write to the bucket. - If it is successful, **Pass** is displayed. Otherwise, check your access configuration. + If it is successful, **The connection is successfully** is displayed. Otherwise, check your access configuration. -4. In the upper-right corner, toggle the audit setting to **On**. +4. Click **Enable** to enable audit logging for the cluster. - TiDB Cloud is ready to write audit logs for the specified cluster to your Amazon S3 bucket. + TiDB Cloud is ready to write audit logs for the specified cluster to your GCS bucket. > **Note:** > -> - After enabling audit logging, if you make any new changes to bucket URI or location, you must click **Restart** to load the changes and rerun the **Test Connectivity** check to make the changes effective. -> - To remove GCS access from TiDB Cloud, simply delete the principal that you added. +> - After enabling audit logging, if you make any new changes to the bucket URI or location, you must click **Test Connection** again to verify that TiDB Cloud can connect to the bucket. Then, click **Enable** to apply the changes. +> - To remove TiDB Cloud's access to your GCS bucket, delete the trust policy granted to this cluster in the Google Cloud console. ## Specify auditing filter rules diff --git a/tidb-cloud/tidb-cloud-billing-dm.md b/tidb-cloud/tidb-cloud-billing-dm.md index f083eb212bc29..5ab94cd92e871 100644 --- a/tidb-cloud/tidb-cloud-billing-dm.md +++ b/tidb-cloud/tidb-cloud-billing-dm.md @@ -11,16 +11,21 @@ This document describes the billing for Data Migration in TiDB Cloud. TiDB Cloud measures the capacity of Data Migration in Replication Capacity Units (RCUs). When you create a Data Migration job, you can select an appropriate specification. The higher the RCU, the better the migration performance. You will be charged for these Data Migration RCUs. -The following table lists the specifications and corresponding performances for Data Migration. +The following table lists the corresponding performance and the maximum number of tables that each Data Migration specification can migrate. -| Specification | Full data migration | Incremental data migration | -|---------------|---------------------|----------------------------| -| 2 RCUs | 25 MiB/s | 10,000 rows/s| -| 4 RCUs | 35 MiB/s | 20,000 rows/s| -| 8 RCUs | 40 MiB/s | 40,000 rows/s| -| 16 RCUs | 45 MiB/s | 80,000 rows/s| +| Specification | Full data migration | Incremental data migration | Maximum number of tables | +|---------------|---------------------|----------------------------|-----------------------| +| 2 RCUs | 25 MiB/s | 10,000 rows/s | 500 | +| 4 RCUs | 35 MiB/s | 20,000 rows/s | 10000 | +| 8 RCUs | 40 MiB/s | 40,000 rows/s | 30000 | +| 16 RCUs | 45 MiB/s | 80,000 rows/s | 60000 | -Note that all the performance values in this table are maximum performances. It is assumed that there are no performance, network bandwidth, or other bottlenecks in the upstream and downstream databases. The performance values are for reference only and might vary in different scenarios. +For more information about the prices of Data Migration RCUs, see [Data Migration Cost](https://www.pingcap.com/tidb-dedicated-pricing-details/#dm-cost). + +> **Note:** +> +> - If the number of tables to be migrated exceeds the maximum number of tables, the Data Migration job might still run, but the job could become unstable or even fail. +> - All the performance values in this table are maximum and optimal ones. It is assumed that there are no performance, network bandwidth, or other bottlenecks in the upstream and downstream databases. The performance values are for reference only and might vary in different scenarios. The Data Migration job measures full data migration performance in MiB/s. This unit indicates the amount of data (in MiB) that is migrated per second by the Data Migration job. @@ -30,23 +35,23 @@ The Data Migration job measures incremental data migration performance in rows/s To learn about the supported regions and the price of TiDB Cloud for each Data Migration RCU, see [Data Migration Cost](https://www.pingcap.com/tidb-cloud-pricing-details/#dm-cost). -The Data Migration job is in the same region as the target TiDB cluster. +The Data Migration job is in the same region as the target TiDB node. -Note that if you are using AWS PrivateLink or VPC peering connections, and if the source database and the TiDB cluster are not in the same region or not in the same availability zone (AZ), two additional traffic charges will be incurred: cross-region and cross-AZ traffic charges. +Note that if you are using AWS PrivateLink or VPC peering connections, and if the source database and the TiDB node are not in the same region or not in the same availability zone (AZ), two additional traffic charges will be incurred: cross-region and cross-AZ traffic charges. -- If the source database and the TiDB cluster are not in the same region, cross-region traffic charges are incurred when the Data Migration job collects data from the source database. +- If the source database and the TiDB node are not in the same region, cross-region traffic charges are incurred when the Data Migration job collects data from the source database. ![Cross-region traffic charges](/media/tidb-cloud/dm-billing-cross-region-fees.png) -- If the source database and the TiDB cluster are in the same region but in different AZs, cross-AZ traffic charges are incurred when the Data Migration job collects data from the source database. +- If the source database and the TiDB node are in the same region but in different AZs, cross-AZ traffic charges are incurred when the Data Migration job collects data from the source database. ![Cross-AZ traffic charges](/media/tidb-cloud/dm-billing-cross-az-fees.png) -- If the Data Migration job and the TiDB cluster are not in the same AZ, cross-AZ traffic charges are incurred when the Data Migration job writes data to the target TiDB cluster. In addition, if the Data Migration job and the TiDB cluster are not in the same AZ (or region) with the source database, cross-AZ (or cross-region) traffic charges are incurred when the Data Migration job collects data from the source database. +- If the Data Migration job and the TiDB node are not in the same AZ, cross-AZ traffic charges are incurred when the Data Migration job writes data to the target TiDB node. In addition, if the Data Migration job and the TiDB node are not in the same AZ (or region) with the source database, cross-AZ (or cross-region) traffic charges are incurred when the Data Migration job collects data from the source database. ![Cross-region and cross-AZ traffic charges](/media/tidb-cloud/dm-billing-cross-region-and-az-fees.png) -The cross-region and cross-AZ traffic prices are the same as those for TiDB Cloud. For more information, see [TiDB Cloud Pricing Details](https://en.pingcap.com/tidb-cloud-pricing-details/). +The cross-region and cross-AZ traffic prices are the same as those for TiDB Cloud. For more information, see [TiDB Cloud Pricing Details](https://www.pingcap.com/tidb-dedicated-pricing-details/). ## See also diff --git a/tidb-cloud/tidb-cloud-billing-recovery-group.md b/tidb-cloud/tidb-cloud-billing-recovery-group.md new file mode 100644 index 0000000000000..4197b257b90c9 --- /dev/null +++ b/tidb-cloud/tidb-cloud-billing-recovery-group.md @@ -0,0 +1,14 @@ +--- +title: Recovery Group Billing +summary: Learn about billing for recovery groups in TiDB Cloud. +--- + +# Recovery Group Billing + +TiDB Cloud bills for recovery groups based on the deployed size of your TiKV nodes in the primary cluster of the recovery group. When you [create a recovery group](/tidb-cloud/recovery-group-get-started.md) for a cluster, you can select the primary cluster for the recovery group. The larger the TiKV configuration, the higher the cost for recovery group protection. + +TiDB Cloud also bills for data processing per GiB basis. The data processing price varies depending on whether the data is replicated to a secondary cluster in another region, or within the same region. + +## Pricing + +To learn about the supported regions and the pricing for TiDB Cloud recovery groups, see [Recovery Group Cost](https://www.pingcap.com/tidb-cloud-pricing-details/#recovery-group-cost). diff --git a/tidb-cloud/tidb-cloud-billing-ticdc-rcu.md b/tidb-cloud/tidb-cloud-billing-ticdc-rcu.md index 14930b63a97ad..adef15f9e404b 100644 --- a/tidb-cloud/tidb-cloud-billing-ticdc-rcu.md +++ b/tidb-cloud/tidb-cloud-billing-ticdc-rcu.md @@ -6,9 +6,11 @@ aliases: ['/tidbcloud/tidb-cloud-billing-tcu'] # Changefeed Billing +## RCU cost + TiDB Cloud measures the capacity of [changefeeds](/tidb-cloud/changefeed-overview.md) in TiCDC Replication Capacity Units (RCUs). When you [create a changefeed](/tidb-cloud/changefeed-overview.md#create-a-changefeed) for a cluster, you can select an appropriate specification. The higher the RCU, the better the replication performance. You will be charged for these TiCDC changefeed RCUs. -## Number of TiCDC RCUs +### Number of TiCDC RCUs The following table lists the specifications and corresponding replication performances for changefeeds: @@ -24,8 +26,14 @@ The following table lists the specifications and corresponding replication perfo > **Note:** > -> The preceding performance data is for reference only and might vary in different scenarios. +> The preceding performance data is for reference only and might vary in different scenarios. It is strongly recommended that you conduct a real workload test before using the changefeed feature in a production environment. For further assistance, contact [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md#get-support-for-a-cluster). -## Price +### Price To learn about the supported regions and the price of TiDB Cloud for each TiCDC RCU, see [Changefeed Cost](https://www.pingcap.com/tidb-cloud-pricing-details/#changefeed-cost). + +## Private Data Link cost + +If you choose the **Private Link** or **Private Service Connect** network connectivity method, additional **Private Data Link** costs will be incurred. These charges fall under the [Data Transfer Cost](https://www.pingcap.com/tidb-dedicated-pricing-details/#data-transfer-cost) category. + +The price of **Private Data Link** is **$0.01/GiB**, the same as **Data Processed** of [AWS Interface Endpoint pricing](https://aws.amazon.com/privatelink/pricing/#Interface_Endpoint_pricing) and **Consumer data processing** of [Google Cloud Private Service Connect pricing](https://cloud.google.com/vpc/pricing#psc-forwarding-rules). diff --git a/tidb-cloud/tidb-cloud-billing.md b/tidb-cloud/tidb-cloud-billing.md index 081904c7cd582..fa1213f8169ed 100644 --- a/tidb-cloud/tidb-cloud-billing.md +++ b/tidb-cloud/tidb-cloud-billing.md @@ -7,12 +7,12 @@ summary: Learn about TiDB Cloud billing. > **Note:** > -> [TiDB Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-serverless) are free until May 31, 2023, with a 100% discount off. After that, usage beyond the [free quota](/tidb-cloud/select-cluster-tier.md#usage-quota) will be charged. +> [TiDB Cloud Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) are free until May 31, 2023, with a 100% discount off. After that, usage beyond the [free quota](/tidb-cloud/select-cluster-tier.md#usage-quota) will be charged. TiDB Cloud charges according to the resources that you consume. You can visit the following pages to get more information about the pricing. -- [TiDB Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details) -- [TiDB Dedicated Pricing Details](https://en.pingcap.com/tidb-cloud-pricing-details/) +- [TiDB Cloud Serverless Pricing Details](https://www.pingcap.com/tidb-serverless-pricing-details/) +- [TiDB Cloud Dedicated Pricing Details](https://www.pingcap.com/tidb-dedicated-pricing-details/) ## Invoices @@ -46,7 +46,9 @@ To view the list of invoices, perform the following steps: > > If you are in multiple organizations, switch to your target organization by clicking its name. -2. Click **Billing**. The invoices page is displayed. +2. In the left navigation pane, click the **Billing** tab. + +3. Click the **Invoices** tab. The invoices page is displayed. ## Billing details @@ -78,6 +80,48 @@ The billing details page shows the billing summary by project and by service. Yo > - The total amount in the monthly bill is rounded off to the 2nd decimal place. > - The total amount in the daily usage details is accurate to the 6th decimal place. +## Cost explorer + +If you are in the `Organization Owner` or `Organization Billing Admin` role of your organization, you can view and analyze the usage costs of TiDB Cloud. Otherwise, skip this section. + +To analyze and customize your cost reports of your organization, perform the following steps: + +1. In the lower-left corner of the [TiDB Cloud console](https://tidbcloud.com), click , and then click **Billing**. + + > **Note:** + > + > If you are in multiple organizations, switch to your target organization by clicking its name. + +2. On the **Billing** page, click the **Cost Explorer** tab. +3. On the **Cost Explorer** page, expand the **Filter** section in the upper-right corner to customize your report. You can set the time range, select a grouping option (such as by service, project, cluster, region, product type, and charge type), and apply filters by selecting specific services, projects, clusters, or regions. The cost explorer will display you with the following information: + + - **Cost Graph**: visualizes the cost trends over the selected time range. You can switch between **Monthly**, **Daily**, and **Total** views. + - **Cost Breakdown**: displays a detailed breakdown of your costs according to the selected grouping option. For further analysis, you can download the data in CSV format. + +## Billing profile + +Paid organizations can create a billing profile. Information in this profile will be used to determine the tax calculation. + +To view or update the billing profile of your organization, click in the lower-left corner and then click **Billing** > **Billing Profile**. + +There are four fields in the billing profile. + +### Company name (optional) + +If this field is specified, this name will appear on invoices instead of your organization name. + +### Billing email (optional) + +If this field is specified, invoices and other billing-related notifications will be sent to this email address. + +### Primary business address + +This is the address of the company that purchases TiDB Cloud services. It is used to calculate any applicable taxes. + +### Business tax ID (optional) + +If your business is registered for VAT/GST, fill in a valid VAT/GST ID. By providing this information, we will exempt you from charging VAT/GST if applicable. This is important for businesses operating in regions where VAT/GST registration allows for certain tax exemptions or refunds. + ## Credits TiDB Cloud offers a certain number of credits for Proof of Concept (PoC) users. One credit is equivalent to one U.S. dollar. You can use credits to pay TiDB cluster fees before the credits become expired. @@ -136,8 +180,6 @@ If you are in the `Organization Owner` or `Organization Billing Admin` role of y > > If you sign up for TiDB Cloud through [AWS Marketplace](https://aws.amazon.com/marketplace) or [Google Cloud Marketplace](https://console.cloud.google.com/marketplace), you can pay through your AWS account or Google Cloud account directly but cannot add payment methods or download invoices in the TiDB Cloud console. -### Add a credit card - The fee is deducted from a bound credit card according to your cluster usage. To add a valid credit card, you can use either of the following methods: - When you are creating a cluster: @@ -156,7 +198,9 @@ The fee is deducted from a bound credit card according to your cluster usage. To 2. Click **Billing**. 3. Under the **Payment Method** tab, click **Add a New Card**. - 4. Fill in the billing address and card information, and then click **Save**. + 4. Fill in the credit card information and credit card address, and then click **Save Card**. + + If you do not specify a primary business address in [**Billing profile**](#billing-profile), the credit card address will be used as your primary business address for tax calculation. You can update your primary business address in **Billing profile** anytime. > **Note:** > @@ -176,22 +220,6 @@ To set the default credit card, perform the following steps: 3. Click the **Payment Method** tab. 4. Select a credit card in the credit card list, and click **Set as default**. -### Edit billing profile information - -The billing profile information includes the business legal address and tax registration information. By providing your tax registration number, certain taxes might be exempted from your invoice. - -To edit the billing profile information, perform the following steps: - -1. Click in the lower-left corner of the TiDB Cloud console. - - > **Note:** - > - > If you are in multiple organizations, switch to your target organization by clicking its name. - -2. Click **Billing**. -3. Click the **Payment Method** tab. -4. Edit the billing profile information, and then click **Save**. - ## Contract If you are in the `Organization Owner` or `Organization Billing Admin` role of your organization, you can manage your customized TiDB Cloud subscriptions in the TiDB Cloud console to meet compliance requirements. Otherwise, skip this section. diff --git a/tidb-cloud/tidb-cloud-budget.md b/tidb-cloud/tidb-cloud-budget.md new file mode 100644 index 0000000000000..66d2a5ecfc470 --- /dev/null +++ b/tidb-cloud/tidb-cloud-budget.md @@ -0,0 +1,113 @@ +--- +title: Manage Budgets for TiDB Cloud +summary: Learn about how to use the budget feature of TiDB Cloud to monitor your costs. +--- + +# Manage Budgets for TiDB Cloud + +In TiDB Cloud, you can use the budget feature to monitor your costs and keep your spending under control. + +When your monthly actual costs exceed the percentage thresholds of your specified budget, alert emails are sent to your organization owners and billing administrators. These notifications help you stay informed and take proactive measures to manage your spending, aligning your expenses with your budget. + +TiDB Cloud provides two types of budgets to help you track your spending: + +- **Serverless Spending Limit** budget: for each TiDB Cloud Serverless scalable cluster, TiDB Cloud automatically creates a **Serverless Spending Limit** budget. This budget helps you track the actual cost against the [spending limit](/tidb-cloud/manage-serverless-spend-limit.md) configured on that cluster. It includes three threshold rules: 75%, 90%, and 100% of the budget, which are not editable. + +- **Custom** budget: you can create custom budgets to track actual costs for an entire organization or specific projects. For each budget, you can specify a budget scope, set a target spending amount, and configure alert thresholds. After creating a custom budget, you can compare your monthly actual costs with your planned costs to ensure you stay within budget. + +## Prerequisites + +To view, create, edit, or delete budgets of your organization or projects, you must be in the `Organization Owner` or `Organization Billing Admin` role of your organization. + +## View the budget information + +To view the budget page of your organization, take the following steps: + +1. In the lower-left corner of the TiDB Cloud console, click , and then click **Billing**. + + > **Note:** + > + > If you are in multiple organizations, switch to your target organization by clicking its name. + +2. On the **Billing** page, click the **Budgets** tab. + +For each budget, you can view its name, type, status, amount used, budget amount, period, and scope. + +## Create a custom budget + +To create a custom budget to monitor the spending of your organization or specific projects, take the following steps: + +1. In the lower-left corner of the TiDB Cloud console, click , and then click **Billing**. + + > **Note:** + > + > If you are in multiple organizations, switch to your target organization by clicking its name. + +2. On the **Billing** page, click the **Budgets** tab. + +3. On the **Budgets** page, click **Create Custom Budget**. You can create up to five custom budgets. + +4. Provide the budget basic settings. + + - **Name**: enter a name for the budget. + - **Period**: select a time range for tracking costs. Currently, you can only select **Monthly**, which starts on the first day of each month and resets at the beginning of each month. TiDB Cloud tracks your actual spending during the time range against your budget amount (your planned spending). + - **Budget scope**: apply the scope to all projects (which means the entire TiDB Cloud organization) or a specific project as needed. + +5. Set the budget amount. + + - **Budget Amount**: enter a planned spending amount for the selected period. + - **Apply credits**: choose whether to apply credits to the running total cost. Credits are used to reduce the cost of your TiDB Cloud usage. When this option is enabled, the budget tracks the running total cost minus credits. + - **Apply discounts**: choose whether to apply discounts to the running total cost. Discounts are reductions in the regular price of TiDB Cloud service. When this option is enabled, the budget tracks the running total cost minus discounts. + +6. Configure alert thresholds for the budget. If your actual spending exceeds specified thresholds during the selected period, TiDB Cloud sends a budget notification email to your organization owners and billing administrators. + + - By default, TiDB Cloud provides three alert thresholds: 75%, 90%, and 100% of the budget amount. You can modify these percentages as needed. + - To add a new alert threshold, click **Add alert threshold.** + - To remove a threshold, click the delete icon next to the threshold. + +7. Click **Create**. + +## Edit a custom budget + +> **Note:** +> +> The **Serverless Spending Limit** budget cannot be edited because it is automatically created by TiDB Cloud to help you track the cost of a TiDB Cloud Serverless scalable cluster against its [spending limit](/tidb-cloud/manage-serverless-spend-limit.md). + +To edit a custom budget, take the following steps: + +1. In the lower-left corner of the TiDB Cloud console, click , and then click **Billing**. + + > **Note:** + > + > If you are in multiple organizations, switch to your target organization by clicking its name. + +2. On the **Billing** page, click the **Budgets** tab. + +3. On the **Budgets** page, locate the row of your budget, click **...** in that row, and then click **Edit**. + +4. Edit the budget name, budget scope, budget amount, and alert thresholds as needed. + + > **Note:** + > + > Editing the budget period and whether to apply credits and discounts is not supported. + +5. Click **Update**. + +## Delete a custom budget + +> **Note:** +> +> - Once a custom budget is deleted, you will no longer receive any alert emails related to it. +> - The **Serverless Spending Limit** budget cannot be deleted because it is automatically created by TiDB Cloud to help you track the cost of a TiDB Cloud Serverless scalable cluster against its [spending limit](/tidb-cloud/manage-serverless-spend-limit.md). + +To delete a custom budget, take the following steps: + +1. In the lower-left corner of the TiDB Cloud console, click , and then click **Billing**. + + > **Note:** + > + > If you are in multiple organizations, switch to your target organization by clicking its name. + +2. On the **Billing** page, click the **Budgets** tab. + +3. On the **Budgets** page, locate the row of your budget, click **...** in that row, and then click **Delete**. \ No newline at end of file diff --git a/tidb-cloud/tidb-cloud-clinic.md b/tidb-cloud/tidb-cloud-clinic.md new file mode 100644 index 0000000000000..6a4c012421911 --- /dev/null +++ b/tidb-cloud/tidb-cloud-clinic.md @@ -0,0 +1,111 @@ +--- +title: TiDB Cloud Clinic +summary: Learn how to use TiDB Cloud Clinic for advanced monitoring and diagnostics. +--- + +# TiDB Cloud Clinic + +TiDB Cloud Clinic offers advanced monitoring and diagnostic capabilities on the TiDB Cloud, designed to help you quickly identify performance issues, optimize your database, and enhance overall performance with detailed analysis and actionable insights. + +![tidb-cloud-clinic](/media/tidb-cloud/tidb-cloud-clinic.png) + +> **Note:** +> +> Currently, TiDB Cloud Clinic is only available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters. + +## Prerequisites + +TiDB Cloud Clinic is only available for organizations that subscribe to the **Enterprise** or **Premium** support plan. + +## View the Cluster page + +To view the **Cluster** page, take the following steps: + +1. Log in to the [TiDB Cloud Clinic console](https://clinic.pingcap.com/) and select **Continue with TiDB Account** to enter the TiDB Cloud login page. + +2. From the organization list, select your target organization. The clusters in the selected project are displayed. + +3. Click the name of your target cluster. The cluster overview page is displayed, where you can view detailed information about your cluster, including: + + - Advanced Metrics + - Top Slow Queries + - TopSQL + - Benchmark Report + +## Monitor advanced metrics + +TiDB Cloud Clinic uses Grafana to provide a comprehensive set of metrics for TiDB clusters. These metrics include dashboards for TiDB, TiKV, TiFlash, BR, Lightning, TiCDC, TiDB resource control, and TiDB performance. + +To view the metrics dashboard, take the following steps: + +1. In the [TiDB Cloud Clinic console](https://clinic.pingcap.com/), navigate to the **Cluster** page of a cluster. + +2. Click **Metrics**. + +3. Click the name of the dashboard you want to view. The dashboard is displayed. + +The retention policy for advanced metrics is 90 days. + +## Analyze top slow queries + +By default, SQL queries that take longer than 300 milliseconds are considered slow queries. + +On the default [**Slow Queries**](/tidb-cloud/tune-performance.md#slow-query) page in the TiDB Cloud console, identifying performance-impacting queries can be difficult, especially in clusters with a large number of slow queries. The **Top Slow Queries** feature in TiDB Cloud Clinic provides aggregated analysis based on slow query logs. With this feature, you can easily pinpoint queries with performance issues, reducing overall performance tuning time by at least half. + +Top Slow Queries displays the top 10 queries aggregated by SQL digest, sorted by the following dimensions: + +- Total latency +- Maximum latency +- Average latency +- Total memory +- Maximum memory +- Average memory +- Total count + +To view slow queries in a cluster, take the following steps: + +1. In the [TiDB Cloud Clinic console](https://clinic.pingcap.com/), navigate to the **Cluster** page of a cluster. + +2. Click **Slow Query**. + +3. The top slow queries are displayed in a table. You can sort the results by different columns. + +4. (Optional) Click any slow query in the list to view its detailed execution information. + +5. (Optional) Filter slow queries by time range, database, or statement type. + +The retention policy for slow queries is 7 days. + +For more information, see [Slow Queries in TiDB Dashboard](https://docs.pingcap.com/tidb/stable/dashboard-slow-query). + +## Monitor TopSQL + +TiDB Cloud Clinic provides TopSQL information, enabling you to monitor and visually explore the CPU overhead of each SQL statement in your database in real time. This helps you optimize and resolve database performance issues. + +To view TopSQL, take the following steps: + +1. In the [TiDB Cloud Clinic console](https://clinic.pingcap.com/), navigate to the **Cluster** page of a cluster. + +2. Click **TopSQL**. + +3. Select a specific TiDB or TiKV instance to observe its load. You can use the time picker or select a time range in the chart to refine your analysis. + +4. Analyze the charts and tables displayed by TopSQL. + +For more information, see [TopSQL in TiDB Dashboard](https://docs.pingcap.com/tidb/stable/top-sql). + +## Generate benchmark reports + +The **Benchmark Report** feature helps you identify performance issues in a TiDB cluster during performance testing. After completing a stress test, you can generate a benchmark report to analyze the cluster's performance. The report highlights identified bottlenecks and provides optimization suggestions. After applying these suggestions, you can run another round of stress testing and generate a new benchmark report to compare performance improvements. + +To generate a benchmark report, take the following steps: + +1. In the [TiDB Cloud Clinic console](https://clinic.pingcap.com/), navigate to the **Cluster** page of a cluster. + +2. Click **Benchmark Report**. + +3. Select the time range to be analyzed in the benchmark report. + +4. Click **Create Report** to generate the benchmark report. + +5. Wait for report generation to complete. When the report is ready, click **View** to open it. diff --git a/tidb-cloud/tidb-cloud-connect-aws-dms.md b/tidb-cloud/tidb-cloud-connect-aws-dms.md new file mode 100644 index 0000000000000..6ad1a127fd72c --- /dev/null +++ b/tidb-cloud/tidb-cloud-connect-aws-dms.md @@ -0,0 +1,140 @@ +--- +title: Connect AWS DMS to TiDB Cloud clusters +summary: Learn how to migrate data from or into TiDB Cloud using AWS Database Migration Service (AWS DMS). +--- + +# Connect AWS DMS to TiDB Cloud clusters + +[AWS Database Migration Service (AWS DMS)](https://aws.amazon.com/dms/) is a cloud service that makes it possible to migrate relational databases, data warehouses, NoSQL databases, and other types of data stores. You can use AWS DMS to migrate your data from or into TiDB Cloud clusters. This document describes how to connect AWS DMS to a TiDB Cloud cluster. + +## Prerequisites + +### An AWS account with enough access + +You are expected to have an AWS account with enough access to manage DMS-related resources. If not, refer to the following AWS documents: + +- [Sign up for an AWS account](https://docs.aws.amazon.com/dms/latest/userguide/CHAP_GettingStarted.SettingUp.html#sign-up-for-aws) +- [Identity and access management for AWS Database Migration Service](https://docs.aws.amazon.com/dms/latest/userguide/security-iam.html) + +### A TiDB Cloud account and a TiDB cluster + +You are expected to have a TiDB Cloud account and a TiDB Cloud Serverless or TiDB Cloud Dedicated cluster. If not, refer to the following documents to create one: + +- [Create a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) +- [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md) + +## Configure network + +Before creating DMS resources, you need to configure network properly to ensure DMS can communicate with TiDB Cloud clusters. If you are unfamiliar with AWS, contact AWS Support. The following provides several possible configurations for your reference. + + + +
    + +For TiDB Cloud Serverless, your clients can connect to clusters via public endpoint or private endpoint. + +- To [connect to a TiDB Cloud Serverless cluster via public endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md), do one of the following to make sure that the DMS replication instance can access the internet. + + - Deploy the replication instance in public subnets and enable **Public accessible**. For more information, see [Configuration for internet access](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Internet_Gateway.html#vpc-igw-internet-access). + + - Deploy the replication instance in private subnets and route traffic in the private subnets to public subnets. In this case, you need at least three subnets, two private subnets, and one public subnet. The two private subnets form a subnet group where the replication instance lives. Then you need to create a NAT gateway in the public subnet and route traffic of the two private subnets to the NAT gateway. For more information, see [Access the internet from a private subnet](https://docs.aws.amazon.com/vpc/latest/userguide/nat-gateway-scenarios.html#public-nat-internet-access). + +- To connect to a TiDB Cloud Serverless cluster via private endpoint, [set up a private endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md) first and deploy the replication instance in private subnets. + +
    + +
    + +For TiDB Cloud Dedicated, your clients can connect to clusters via public endpoint, private endpoint, or VPC peering. + +- To [connect to a TiDB Cloud Dedicated cluster via public endpoint](/tidb-cloud/connect-via-standard-connection.md), do one of the following to make sure that the DMS replication instance can access the internet. In addition, you need to add the public IP address of the replication instance or NAT gateway to the cluster's [IP access list](/tidb-cloud/configure-ip-access-list.md). + + - Deploy the replication instance in public subnets and enable **Public accessible**. For more information, see [Configuration for internet access](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Internet_Gateway.html#vpc-igw-internet-access). + + - Deploy the replication instance in private subnets and route traffic in the private subnets to public subnets. In this case, you need at least three subnets, two private subnets, and one public subnet. The two private subnets form a subnet group where the replication instance lives. Then you need to create a NAT gateway in the public subnet and route traffic of the two private subnets to the NAT gateway. For more information, see [Access the internet from a private subnet](https://docs.aws.amazon.com/vpc/latest/userguide/nat-gateway-scenarios.html#public-nat-internet-access). + +- To connect to a TiDB Cloud Dedicated cluster via private endpoint, [set up a private endpoint](/tidb-cloud/set-up-private-endpoint-connections.md) first and deploy the replication instance in private subnets. + +- To connect to a TiDB Cloud Dedicated cluster via VPC peering, [set up a VPC peering connection](/tidb-cloud/set-up-vpc-peering-connections.md) first and deploy the replication instance in private subnets. + +
    +
    + +## Create an AWS DMS replication instance + +1. In the AWS DMS console, go to the [**Replication instances**](https://console.aws.amazon.com/dms/v2/home#replicationInstances) page and switch to the corresponding region. It is recommended to use the same region for AWS DMS as TiDB Cloud. + + ![Create replication instance](/media/tidb-cloud/aws-dms-tidb-cloud/aws-dms-connect-replication-instances.png) + +2. Click **Create replication instance**. + +3. Fill in an instance name, ARN, and description. + +4. In the **Instance configuration** section, configure the instance: + - **Instance class**: select an appropriate instance class. For more information, see [Choosing replication instance types](https://docs.aws.amazon.com/dms/latest/userguide/CHAP_ReplicationInstance.Types.html). + - **Engine version**: keep the default configuration. + - **High Availability**: select **Multi-AZ** or **Single-AZ** based on your business needs. + +5. Configure the storage in the **Allocated storage (GiB)** field. + +6. Configure connectivity and security. You can refer to [the previous section](#configure-network) for network configuration. + + - **Network type - new**: select **IPv4**. + - **Virtual private cloud (VPC) for IPv4**: select the VPC that you need. + - **Replication subnet group**: select a subnet group for your replication instance. + - **Public accessible**: set it based on your network configuration. + + ![Connectivity and security](/media/tidb-cloud/aws-dms-tidb-cloud/aws-dms-connect-connectivity-security.png) + +7. Configure the **Advanced settings**, **Maintenance**, and **Tags** sections if needed, and then click **Create replication instance** to finish the instance creation. + +> **Note:** +> +> AWS DMS also supports serverless replications. For detailed steps, see [Creating a serverless replication](https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Serverless.Components.html#CHAP_Serverless.create). Unlike replication instances, AWS DMS serverless replications do not provide the **Public accessible** option. + +## Create TiDB Cloud DMS endpoints + +For connectivity, the steps for using TiDB Cloud clusters as a source or as a target are similar, but DMS does have some different database setting requirements for source and target. For more information, see [Using MySQL as a source](https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Source.MySQL.html) or [Using MySQL as a target](https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Target.MySQL.html). When using a TiDB Cloud cluster as a source, you can only **Migrate existing data** because TiDB does not support MySQL binlog. + +1. In the AWS DMS console, go to the [**Endpoints**](https://console.aws.amazon.com/dms/v2/home#endpointList) page and switch to the corresponding region. + + ![Create endpoint](/media/tidb-cloud/aws-dms-tidb-cloud/aws-dms-connect-create-endpoint.png) + +2. Click **Create endpoint** to create the target database endpoint. + +3. In the **Endpoint type** section, select **Source endpoint** or **Target endpoint**. + +4. In the **Endpoint configuration** section, fill in the **Endpoint identifier** and ARN fields. Then, select **MySQL** as **Source engine** or **Target engine**. + +5. For the **Access to endpoint database** field, select the **Provide access information manually** checkbox and fill in cluster information as follows: + + + +
    + + - **Server name**: `HOST` of TiDB Cloud Serverless cluster. + - **Port**: `PORT` of TiDB Cloud Serverless cluster. + - **User name**: User of TiDB Cloud Serverless cluster for migration. Make sure it meets DMS requirements. + - **Password**: Password of the TiDB Cloud Serverless cluster user. + - **Secure Socket Layer (SSL) mode**: If you are connecting via public endpoint, it is highly recommended to set the mode to **verify-full** to ensure transport security. If you are connecting via private endpoint, you can set the mode to **none**. + - (Optional) **CA certificate**: Use the [ISRG Root X1 certificate](https://letsencrypt.org/certs/isrgrootx1.pem). For more information, see [TLS Connections to TiDB Cloud Serverless](/tidb-cloud/secure-connections-to-serverless-clusters.md). + +
    + +
    + + - **Server name**: `HOST` of TiDB Cloud Dedicated cluster. + - **Port**: `PORT` of TiDB Cloud Dedicated cluster. + - **User name**: User of TiDB Cloud Dedicated cluster for migration. Make sure it meets DMS requirements. + - **Password**: Password of TiDB Cloud Dedicated cluster user. + - **Secure Socket Layer (SSL) mode**: If you are connecting via public endpoint, it is highly recommended to set the mode to **verify-full** to ensure transport security. If you are connecting via private endpoint, you can set it to **none**. + - (Optional) **CA certificate**: Get the CA certificate according to [TLS connections to TiDB Cloud Dedicated](/tidb-cloud/tidb-cloud-tls-connect-to-dedicated.md). + +
    +
    + + ![Provide access information manually](/media/tidb-cloud/aws-dms-tidb-cloud/aws-dms-connect-configure-endpoint.png) + +6. If you want to create the endpoint as a **Target endpoint**, expand the **Endpoint settings** section, select the **Use endpoint connection attributes** checkbox, and then set **Extra connection attributes** to `Initstmt=SET FOREIGN_KEY_CHECKS=0;`. + +7. Configure the **KMS Key** and **Tags** sections if needed. Click **Create endpoint** to finish the instance creation. diff --git a/tidb-cloud/tidb-cloud-console-auditing.md b/tidb-cloud/tidb-cloud-console-auditing.md index b319e5952e21f..0555b8dfad2f6 100644 --- a/tidb-cloud/tidb-cloud-console-auditing.md +++ b/tidb-cloud/tidb-cloud-console-auditing.md @@ -9,7 +9,7 @@ TiDB Cloud provides the console audit logging feature to help you track various ## Prerequisites -- You must be in the `Organization Owner` or `Organization Console Audit Admin` role of your organization in TiDB Cloud. Otherwise, you cannot see the console audit logging-related options in the TiDB Cloud console. The `Organization Console Audit Admin` role is only visible upon request, so it is recommended that you use the `Organization Owner` role directly. If you need to use the `Organization Console Audit Admin` role, click **?** in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com) and click **Request Support**. Then, fill in "Apply for the Organization Console Audit Admin role" in the **Description** field and click **Send**. For more information about roles in TiDB Cloud, see [User roles](/tidb-cloud/manage-user-access.md#user-roles). +- You must be in the `Organization Owner` or `Organization Console Audit Admin` role of your organization in TiDB Cloud. Otherwise, you cannot see the console audit logging-related options in the TiDB Cloud console. - You can only enable and disable the console audit logging for your organization. You can only track the actions of users in your organization. - After the console audit logging is enabled, all event types of the TiDB Cloud console will be audited, and you cannot specify only auditing some of them. @@ -121,7 +121,7 @@ The console audit logs record various user activities on the TiDB Cloud console | PauseCluster | Pause a cluster | | ResumeCluster | Resume a cluster | | ScaleCluster | Scale a cluster | -| DownloadTiDBClusterCA | Download TiDB cluster CA certificate | +| DownloadTiDBClusterCA | Download CA certificate | | OpenWebSQLConsole | Connect to a TiDB cluster through Web SQL | | SetRootPassword | Set the root password of a TiDB cluster | | UpdateIPAccessList | Update the IP access list of a TiDB cluster | @@ -147,12 +147,12 @@ The console audit logs record various user activities on the TiDB Cloud console | BindSupportPlan | Bind a support plan | | CancelSupportPlan | Cancel a support plan | | UpdateOrganizationName | Update the organization name | -| SetSpendLimit | Edit the spending limit of a TiDB Serverless cluster | +| SetSpendLimit | Edit the spending limit of a TiDB Cloud Serverless scalable cluster | | UpdateMaintenanceWindow | Modify maintenance window start time | | DeferMaintenanceTask | Defer a maintenance task | -| CreateBranch | Create a TiDB Serverless branch | -| DeleteBranch | Delete a TiDB Serverless branch | -| SetBranchRootPassword | Set root password for a TiDB Serverless branch | +| CreateBranch | Create a TiDB Cloud Serverless branch | +| DeleteBranch | Delete a TiDB Cloud Serverless branch | +| SetBranchRootPassword | Set root password for a TiDB Cloud Serverless branch | | ConnectBranchGitHub | Connect the cluster with a GitHub repository to enable branching integration | | DisconnectBranchGitHub | Disconnect the cluster from a GitHub repository to disable branching integration | diff --git a/tidb-cloud/tidb-cloud-dm-precheck-and-troubleshooting.md b/tidb-cloud/tidb-cloud-dm-precheck-and-troubleshooting.md index 3e9417f260cee..7f419a96114d7 100644 --- a/tidb-cloud/tidb-cloud-dm-precheck-and-troubleshooting.md +++ b/tidb-cloud/tidb-cloud-dm-precheck-and-troubleshooting.md @@ -88,6 +88,12 @@ The TiDB cluster storage is running low. It is recommended to [increase the TiKV Failed to connect to the source database. It is recommended to check whether the source database is started, the number of database connections has not reached the upper limit, and you can connect using the parameters specified by the job. After confirming that the source database is available, you can try to resume the job by clicking **Restart**. +### Error message: "Error 1273: Unsupported collation when new collation is enabled: 'utf8mb4_0900_ai_ci'" + +Failed to create a schema in the downstream TiDB cluster. This error means that the collation used by the upstream MySQL is not supported by the TiDB cluster. + +To resolve this issue, you can create a schema in the TiDB cluster based on a [supported collation](/character-set-and-collation.md#character-sets-and-collations-supported-by-tidb), and then resume the task by clicking **Restart**. + ## Alerts You can subscribe to TiDB Cloud alert emails to be informed in time when an alert occurs. diff --git a/tidb-cloud/tidb-cloud-encrypt-cmek.md b/tidb-cloud/tidb-cloud-encrypt-cmek.md index 361d0b8e34fc8..e8231c3d3bf2f 100644 --- a/tidb-cloud/tidb-cloud-encrypt-cmek.md +++ b/tidb-cloud/tidb-cloud-encrypt-cmek.md @@ -5,7 +5,7 @@ summary: Learn about how to use Customer-Managed Encryption Key (CMEK) in TiDB C # Encryption at Rest Using Customer-Managed Encryption Keys -Customer-Managed Encryption Key (CMEK) allows you to secure your static data in a TiDB Dedicated cluster by utilizing a cryptographic key that is under your complete control. This key is referred to as the CMEK key. +Customer-Managed Encryption Key (CMEK) allows you to secure your static data in a TiDB Cloud Dedicated cluster by utilizing a symmetric encryption key that is under your complete control. This key is referred to as the CMEK key. Once CMEK is enabled for a project, all clusters created within that project encrypt their static data using the CMEK key. Additionally, any backup data generated by these clusters is encrypted using the same key. If CMEK is not enabled, TiDB Cloud employs an escrow key to encrypt all data in your cluster when it is at rest. @@ -17,7 +17,7 @@ Once CMEK is enabled for a project, all clusters created within that project enc - Currently, TiDB Cloud only supports using AWS KMS to provide CMEK. - To use CMEK, you need to enable CMEK when creating a project and complete CMEK-related configurations before creating a cluster. You cannot enable CMEK for existing projects. -- Currently, in CMEK-enabled projects, you can only create [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters hosted on AWS. TiDB Dedicated clusters hosted on Google Cloud and [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters are not supported. +- Currently, in CMEK-enabled projects, you can only create [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters hosted on AWS. TiDB Cloud Dedicated clusters hosted on Google Cloud and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters are not supported. - Currently, for a specific project, you can only enable CMEK for one AWS region. Once you have configured it, you cannot create clusters in other regions within the same project. ## Enable CMEK @@ -34,8 +34,8 @@ If you are in the `Organization Owner` role of your organization, you can create To create a CMEK-enabled project, take the following steps: 1. Click in the lower-left corner of the TiDB Cloud console. -2. Click **Organization Settings**. -3. On the **Organization Settings** page, click **Create New Project** to open the project creation dialog. +2. Click **Organization Settings**, and then click the **Projects** tab in the left navigation pane. The **Projects** tab is displayed. +3. Click **Create New Project** to open the project creation dialog. 4. Fill in a project name. 5. Choose to enable the CMEK capability of the project. 6. Click **Confirm** to complete the project creation. @@ -67,7 +67,7 @@ To complete the CMEK configuration of the project, take the following steps: 2. Click **Encryption Access** to enter the encryption management page of the project. 3. Click **Create Encryption Key** to enter the key creation page. 4. The key provider only supports AWS KMS. You can choose the region where the encryption key can be used. -5. Copy and save the JSON file as `ROLE-TRUST-POLICY.JSON`. This file describes the trust relationship. +5. Copy and save the JSON file as `ROLE-TRUST-POLICY.JSON`. This file describes the trust relationship. 6. Add this trust relationship to the key policy of AWS KMS. For more information, refer to [Key policies in AWS KMS](https://docs.aws.amazon.com/kms/latest/developerguide/key-policies.html). 7. In the TiDB Cloud console, scroll to the bottom of the key creation page, and then fill in the **KMS Key ARN** obtained from AWS KMS. 8. Click **Create** to create the key. @@ -141,7 +141,7 @@ To complete the CMEK configuration of the project, take the following steps: ### Step 3. Create a cluster -Under the project created in [Step 1](#step-1-create-a-cmek-enabled-project), create a TiDB Dedicated cluster hosted on AWS. For detailed steps, refer to [this document](/tidb-cloud/create-tidb-cluster.md). Ensure that the region where the cluster is located is the same as that in [Step 2](/tidb-cloud/tidb-cloud-encrypt-cmek.md#step-2-complete-the-cmek-configuration-of-the-project). +Under the project created in [Step 1](#step-1-create-a-cmek-enabled-project), create a TiDB Cloud Dedicated cluster hosted on AWS. For detailed steps, refer to [this document](/tidb-cloud/create-tidb-cluster.md). Ensure that the region where the cluster is located is the same as that in [Step 2](/tidb-cloud/tidb-cloud-encrypt-cmek.md#step-2-complete-the-cmek-configuration-of-the-project). > **Note:** > diff --git a/tidb-cloud/tidb-cloud-events.md b/tidb-cloud/tidb-cloud-events.md index 2d826782cbeaf..d56592d4174d4 100644 --- a/tidb-cloud/tidb-cloud-events.md +++ b/tidb-cloud/tidb-cloud-events.md @@ -9,10 +9,6 @@ TiDB Cloud logs the historical events at the cluster level. An *event* indicates This document describes how to view the events for TiDB Cloud clusters using the **Events** page and lists the supported event types. -> **Note:** -> -> Currently, the Events page is only available for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. - ## View the Events page To view the events on the Events page, take the following steps: @@ -46,6 +42,8 @@ TiDB Cloud logs the following types of cluster events: | ScaleChangefeed | Scale the specification of a changefeed | | FailedChangefeed | Changefeed failures | | ImportData | Import data to a cluster | +| UpdateSpendingLimit | Update spending limit of a TiDB Cloud Serverless scalable cluster | +| ResourceLimitation | Update resource limitation of a TiDB Cloud Serverless cluster | For each event, the following information is logged: @@ -55,10 +53,6 @@ For each event, the following information is logged: - Time - Triggered By -> **Note:** -> -> Cluster events started before 2023-03-22 are not visible on the Events page. - ## Event retention policy -For TiDB Dedicated clusters, the event data is kept for 7 days. +Event data is kept for 7 days. diff --git a/tidb-cloud/tidb-cloud-faq.md b/tidb-cloud/tidb-cloud-faq.md index 2a60a3df0d3c7..94d298725e660 100644 --- a/tidb-cloud/tidb-cloud-faq.md +++ b/tidb-cloud/tidb-cloud-faq.md @@ -19,13 +19,13 @@ TiDB Cloud allows developers and DBAs with little or no training to handle once- ### What is the relationship between TiDB and TiDB Cloud? -TiDB is an open-source database and is the best option for organizations who want to run TiDB Self-Hosted in their own data centers, in a self-managed cloud environment, or in a hybrid of the two. +TiDB is an open-source database and is the best option for organizations who want to run TiDB Self-Managed in their own data centers, in a self-managed cloud environment, or in a hybrid of the two. TiDB Cloud is a fully managed cloud Database as a Service of TiDB. It has an easy-to-use web-based management console to let you manage TiDB clusters for mission-critical production environments. ### Is TiDB Cloud compatible with MySQL? -Currently, TiDB Cloud supports the majority of MySQL 5.7 syntax with the exception of triggers, stored procedures, user-defined functions, and foreign keys. For more details, see [Compatibility with MySQL](https://docs.pingcap.com/tidb/stable/mysql-compatibility). +Currently, TiDB Cloud supports the majority of MySQL 5.7 and MySQL 8.0 syntax with the exception of triggers, stored procedures, and user-defined functions. For more details, see [Compatibility with MySQL](/mysql-compatibility.md). ### What programming languages can I use to work with TiDB Cloud? @@ -41,18 +41,18 @@ No. ### What versions of TiDB are supported on TiDB Cloud? -- Starting from July 25, 2023, the default TiDB version for new TiDB Dedicated clusters is v7.1.1. -- Starting from March 7, 2023, the default TiDB version for new TiDB Serverless clusters is v6.6.0. +- Starting from November 26, 2024, the default TiDB version for new TiDB Cloud Dedicated clusters is v8.1.1. +- Starting from February 21, 2024, the TiDB version for TiDB Cloud Serverless clusters is v7.1.3. For more information, see [TiDB Cloud Release Notes](/tidb-cloud/tidb-cloud-release-notes.md). ### What companies are using TiDB or TiDB Cloud in production? -TiDB is trusted by over 1500 global enterprises across a variety of industries, such as financial services, gaming, and e-commerce. Our users include Square (US), Shopee (Singapore), and China UnionPay (China). See our [case studies](https://en.pingcap.com/customers/) for specific details. +TiDB is trusted by over 1500 global enterprises across a variety of industries, such as financial services, gaming, and e-commerce. Our users include Square (US), Shopee (Singapore), and China UnionPay (China). See our [case studies](https://www.pingcap.com/customers/) for specific details. ### What does the SLA look like? -TiDB Cloud provides 99.99% SLA. For details, see [Service Level Agreement for TiDB Cloud Services](https://en.pingcap.com/legal/service-level-agreement-for-tidb-cloud-services/). +TiDB Cloud provides 99.99% SLA. For details, see [Service Level Agreement for TiDB Cloud Services](https://www.pingcap.com/legal/service-level-agreement-for-tidb-cloud-services/). ### How can I learn more about TiDB Cloud? @@ -60,7 +60,7 @@ The best way to learn about TiDB Cloud is to follow our step-by-step tutorial. C - [TiDB Cloud Introduction](/tidb-cloud/tidb-cloud-intro.md) - [Get Started](/tidb-cloud/tidb-cloud-quickstart.md) -- [Create a TiDB Serverless Cluster](/tidb-cloud/create-tidb-cluster-serverless.md) +- [Create a TiDB Cloud Serverless Cluster](/tidb-cloud/create-tidb-cluster-serverless.md) ### What does `XXX's Org/default project/Cluster0` refer to when deleting a cluster? @@ -92,7 +92,7 @@ Each data change is recorded as a Raft log. Through Raft log replication, data i TiDB uses the Raft consensus algorithm to ensure that data is highly available and safely replicated throughout storage in Raft Groups. Data is redundantly copied between TiKV nodes and placed in different Availability Zones to protect against machine or data center failure. With automatic failover, TiDB ensures that your service is always on. -As a Software as a Service (SaaS) provider, we take data security seriously. We have established strict information security policies and procedures required by the [Service Organization Control (SOC) 2 Type 1 compliance](https://en.pingcap.com/press-release/pingcap-successfully-completes-soc-2-type-1-examination-for-tidb-cloud/). This ensures that your data is secure, available, and confidential. +As a Software as a Service (SaaS) provider, we take data security seriously. We have established strict information security policies and procedures required by the [Service Organization Control (SOC) 2 Type 1 compliance](https://www.pingcap.com/press-release/pingcap-successfully-completes-soc-2-type-1-examination-for-tidb-cloud/). This ensures that your data is secure, available, and confidential. ## Migration FAQ @@ -104,7 +104,7 @@ TiDB is highly compatible with MySQL. You can migrate data from any MySQL-compat ### Does TiDB Cloud support incremental backups? -No. If you need to restore data to any point in time within the cluster's backup retention, you can use PITR (Point-in-time Recovery). For more information, see [Use PITR in a TiDB Dedicated cluster](/tidb-cloud/backup-and-restore.md#automatic-backup) or [Use PITR in a TiDB Serverless cluster](/tidb-cloud/backup-and-restore-serverless.md#restore). +No. If you need to restore data to any point in time within the cluster's backup retention, you can use PITR (Point-in-time Recovery). For more information, see [Use PITR in a TiDB Cloud Dedicated cluster](/tidb-cloud/backup-and-restore.md#turn-on-auto-backup) or [Use PITR in a TiDB Cloud Serverless cluster](/tidb-cloud/backup-and-restore-serverless.md#restore). ## HTAP FAQs @@ -149,16 +149,16 @@ No. TiDB Cloud is Database-as-a-Service (DBaaS) and runs only in the TiDB Cloud ### Is my TiDB cluster secure? -In TiDB Cloud, you can use either a TiDB Dedicated cluster or a TiDB Serverless cluster according to your needs. +In TiDB Cloud, you can use either a TiDB Cloud Dedicated cluster or a TiDB Cloud Serverless cluster according to your needs. -For TiDB Dedicated clusters, TiDB Cloud ensures cluster security with the following measures: +For TiDB Cloud Dedicated clusters, TiDB Cloud ensures cluster security with the following measures: - Creates independent sub-accounts and VPCs for each cluster. - Sets up firewall rules to isolate external connections. - Creates server-side TLS certificates and component-level TLS certificates for each cluster to encrypt cluster data in transit. - Provide IP access rules for each cluster to ensure that only allowed source IP addresses can access your cluster. -For TiDB Serverless clusters, TiDB Cloud ensures cluster security with the following measures: +For TiDB Cloud Serverless clusters, TiDB Cloud ensures cluster security with the following measures: - Creates independent sub-accounts for each cluster. - Sets up firewall rules to isolate external connections. @@ -167,27 +167,27 @@ For TiDB Serverless clusters, TiDB Cloud ensures cluster security with the follo ### How do I connect to my database in a TiDB cluster? -
    +
    -For a TiDB Dedicated cluster, the steps to connect to your cluster are simplified as follows: +For a TiDB Cloud Dedicated cluster, the steps to connect to your cluster are simplified as follows: 1. Authorize your network. 2. Set up your database users and login credentials. 3. Download and configure TLS for your cluster server. 4. Choose a SQL client, get an auto-generated connection string displayed on the TiDB Cloud UI, and then connect to your cluster through the SQL client using the string. -For more information, see [Connect to Your TiDB Dedicated Cluster](/tidb-cloud/connect-to-tidb-cluster.md). +For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](/tidb-cloud/connect-to-tidb-cluster.md).
    -
    +
    -For a TiDB Serverless cluster, the steps to connect to your cluster are simplified as follows: +For a TiDB Cloud Serverless cluster, the steps to connect to your cluster are simplified as follows: 1. Set a database user and login credential. 2. Choose a SQL client, get an auto-generated connection string displayed on the TiDB Cloud UI, and then connect to your cluster through the SQL client using the string. -For more information, see [Connect to Your TiDB Serverless Cluster](/tidb-cloud/connect-to-tidb-cluster-serverless.md). +For more information, see [Connect to Your TiDB Cloud Serverless Cluster](/tidb-cloud/connect-to-tidb-cluster-serverless.md).
    @@ -197,3 +197,7 @@ For more information, see [Connect to Your TiDB Serverless Cluster](/tidb-cloud/ ### What support is available for customers? TiDB Cloud is supported by the same team behind TiDB, which has run mission-critical use cases for over 1500 global enterprises across industries including financial services, e-commerce, enterprise applications, and gaming. TiDB Cloud offers a free basic support plan for each user and you can upgrade to a paid plan for extended services. For more information, see [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). + +### How do I check if TiDB Cloud is down? + +You can check the current uptime status of TiDB Cloud on the [System Status](https://status.tidbcloud.com/) page. \ No newline at end of file diff --git a/tidb-cloud/tidb-cloud-glossary.md b/tidb-cloud/tidb-cloud-glossary.md index 95b9a594517cb..6ce89536cc7cf 100644 --- a/tidb-cloud/tidb-cloud-glossary.md +++ b/tidb-cloud/tidb-cloud-glossary.md @@ -25,11 +25,9 @@ ACID refers to the four key properties of a transaction: atomicity, consistency, ### Chat2Query -TiDB Cloud is powered by AI. You can use Chat2Query (beta), an AI-powered SQL editor in the [TiDB Cloud console](https://tidbcloud.com/), to maximize your data value. +Chat2Query is an AI-powered feature integrated into SQL Editor that assists users in generating, debugging, or rewriting SQL queries using natural language instructions. For more information, see [Explore your data with AI-assisted SQL Editor](/tidb-cloud/explore-data-with-chat2query.md). -In Chat2Query, you can either simply type `--` followed by your instructions to let AI generate SQL queries automatically or write SQL queries manually, and then run SQL queries against databases without a terminal. You can find the query results in tables intuitively and check the query logs easily. For more information, see [Chat2Query (beta)](/tidb-cloud/explore-data-with-chat2query.md). - -In addition, TiDB Cloud provides a Chat2Query API for TiDB Serverless clusters. After it is enabled, TiDB Cloud will automatically create a system Data App called **Chat2Query** and a Chat2Data endpoint in Data Service. You can call this endpoint to let AI generate and execute SQL statements by providing instructions. For more information, see [Get started with Chat2Query API](/tidb-cloud/use-chat2query-api.md). +In addition, TiDB Cloud provides a Chat2Query API for TiDB Cloud Serverless clusters. After it is enabled, TiDB Cloud will automatically create a system Data App called **Chat2Query** and a Chat2Data endpoint in Data Service. You can call this endpoint to let AI generate and execute SQL statements by providing instructions. For more information, see [Get started with Chat2Query API](/tidb-cloud/use-chat2query-api.md). ### Credit @@ -49,6 +47,10 @@ Data Service (beta) enables you to access TiDB Cloud data via an HTTPS request u For more information, see [Data Service Overview](/tidb-cloud/data-service-overview.md). +### Direct Customer + +A direct customer is an end customer who purchases TiDB Cloud and pays invoices directly from PingCAP. This is distinguished from an [MSP customer](#msp-customer). + ## E ### Endpoint @@ -67,6 +69,14 @@ A user that has been invited to an organization, with access to the organization Starting from v5.0, TiDB introduces Massively Parallel Processing (MPP) architecture through TiFlash nodes, which shares the execution workloads of large join queries among TiFlash nodes. When the MPP mode is enabled, TiDB, based on cost, determines whether to use the MPP framework to perform the calculation. In the MPP mode, the join keys are redistributed through the Exchange operation while being calculated, which distributes the calculation pressure to each TiFlash node and speeds up the calculation. For more information, see [Use TiFlash MPP Mode](/tiflash/use-tiflash-mpp-mode.md). +### MSP Customer + +A managed service provider (MSP) customer is an end customer who purchases TiDB Cloud and pays invoices through the MSP channel. This is distinguished from a [direct customer](#direct-customer). + +### Managed Service Provider (MSP) + +A managed service provider (MSP) is a partner who resells TiDB Cloud and provides value-added services, including but not limited to TiDB Cloud organization management, billing services, and technical support. + ## N ### node @@ -101,7 +111,7 @@ Project members are users who are invited to join one or more projects of the or ### Recycle Bin -The place where the data of deleted clusters with valid backups is stored. Once a backed-up cluster is deleted, the existing backup files of the cluster are moved to the recycle bin. For backup files from automatic backups, the recycle bin will retain them for 7 days. For backup files from manual backups, there is no expiration date. To avoid data loss, remember to restore the data to a new cluster in time. Note that if a cluster **has no backup**, the deleted cluster will not be displayed here. +The place where the data of deleted clusters with valid backups is stored. Once a backed-up TiDB Cloud Dedicated cluster is deleted, the existing backup files of the cluster are moved to the recycle bin. For backup files from automatic backups, the recycle bin will retain them for a specified period. You can configure the backup retention in **Backup Settings**, and the default is 7 days. For backup files from manual backups, there is no expiration date. To avoid data loss, remember to restore the data to a new cluster in time. Note that if a cluster **has no backup**, the deleted cluster will not be displayed here. ### region @@ -123,13 +133,13 @@ The replication of changefeed is charged according to the computing resources, w ### Request Unit -A Request Unit (RU) is a unit of measure used to represent the amount of resources consumed by a single request to the database. The amount of RUs consumed by a request depends on various factors, such as the operation type or the amount of data being retrieved or modified. For more information, see [TiDB Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details). +A Request Unit (RU) is a unit of measure used to represent the amount of resources consumed by a single request to the database. The amount of RUs consumed by a request depends on various factors, such as the operation type or the amount of data being retrieved or modified. For more information, see [TiDB Cloud Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details). ## S ### Spending limit -Spending limit refers to the maximum amount of money that you are willing to spend on a particular workload in a month. It is a cost-control mechanism that allows you to set a budget for your TiDB Serverless clusters. When the spending limit of a cluster is greater than 0, the cluster is considered a paid cluster. Also, the paid cluster can have a free quota if it meets the qualifications. The paid cluster with a free quota will consume the free quota first. +Spending limit refers to the maximum amount of money that you are willing to spend on a particular workload in a month. It is a cost-control mechanism that enables you to set a budget for your TiDB Cloud Serverless clusters. For [scalable clusters](/tidb-cloud/select-cluster-tier.md#scalable-cluster-plan), the spending limit must be set to a minimum of $0.01. Also, the scalable cluster can have a free quota if it meets the qualifications. The scalable cluster with a free quota will consume the free quota first. ## T diff --git a/tidb-cloud/tidb-cloud-guide-sample-application-java.md b/tidb-cloud/tidb-cloud-guide-sample-application-java.md deleted file mode 100644 index 66b5b6b64dd7a..0000000000000 --- a/tidb-cloud/tidb-cloud-guide-sample-application-java.md +++ /dev/null @@ -1,1730 +0,0 @@ ---- -title: Build a Simple CRUD App with TiDB and Java -summary: Learn how to build a simple CRUD application with TiDB and Java. ---- - - - - -# Build a Simple CRUD App with TiDB and Java - -This document describes how to use TiDB and Java to build a simple CRUD application. - -> **Note:** -> -> It is recommended to use Java 8 or a later Java version. -> -> If you want to use Spring Boot for application development, refer to [Build the TiDB app using Spring Boot](/develop/dev-guide-sample-application-java-spring-boot.md) - -## Step 1. Launch your TiDB cluster - - - -The following introduces how to start a TiDB cluster. - -**Use a TiDB Serverless cluster** - -For detailed steps, see [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). - -**Use a local cluster** - -For detailed steps, see [Deploy a local test cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a TiDB Cluster Using TiUP](/production-deployment-using-tiup.md). - - - - - -See [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). - - - -## Step 2. Get the code - -```shell -git clone https://github.com/pingcap-inc/tidb-example-java.git -``` - - - -
    - -Compared with [Mybatis](https://mybatis.org/mybatis-3/index.html), the JDBC implementation might be not a best practice, because you need to write error handling logic manually and cannot reuse code easily, which makes your code slightly redundant. - -Mybatis is a popular open-source Java class persistence framework. The following uses [MyBatis Generator](https://mybatis.org/generator/quickstart.html) as a Maven plugin to generate the persistence layer code. - -Change to the `plain-java-mybatis` directory: - -```shell -cd plain-java-mybatis -``` - -The structure of this directory is as follows: - -``` -. -├── Makefile -├── pom.xml -└── src - └── main - ├── java - │   └── com - │   └── pingcap - │   ├── MybatisExample.java - │   ├── dao - │   │   └── PlayerDAO.java - │   └── model - │   ├── Player.java - │   ├── PlayerMapper.java - │   └── PlayerMapperEx.java - └── resources - ├── dbinit.sql - ├── log4j.properties - ├── mapper - │   ├── PlayerMapper.xml - │   └── PlayerMapperEx.xml - ├── mybatis-config.xml - └── mybatis-generator.xml -``` - -The automatically generated files are: - -- `src/main/java/com/pingcap/model/Player.java`: The `Player` entity class. -- `src/main/java/com/pingcap/model/PlayerMapper.java`: The interface of `PlayerMapper`. -- `src/main/resources/mapper/PlayerMapper.xml`: The XML mapping of `Player`. Mybatis uses this configuration to automatically generate the implementation class of the `PlayerMapper` interface. - -The strategy for generating these files is written in `mybatis-generator.xml`, which is the configuration file for [Mybatis Generator](https://mybatis.org/generator/quickstart.html). There are comments in the following configuration file to describe how to use it. - -```xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -``` - -`mybatis-generator.xml` is included in `pom.xml` as the configuration of `mybatis-generator-maven-plugin`. - -```xml - - org.mybatis.generator - mybatis-generator-maven-plugin - 1.4.1 - - src/main/resources/mybatis-generator.xml - true - true - - - - - - mysql - mysql-connector-java - 5.1.49 - - - -``` - -Once included in the Maven plugin, you can delete the old generated files and make new ones using `mvn mybatis-generate`. Or you can use `make gen` to delete the old file and generate a new one at the same time. - -> **Note:** -> -> The property `configuration.overwrite` in `mybatis-generator.xml` only ensures that the generated Java code files are overwritten. But the XML mapping files are still written as appended. Therefore, it is recommended to delete the old file before Mybaits Generator generating a new one. - -`Player.java` is a data entity class file generated using Mybatis Generator, which is a mapping of database tables in the application. Each property of the `Player` class corresponds to a field in the `player` table. - -```java -package com.pingcap.model; - -public class Player { - private String id; - - private Integer coins; - - private Integer goods; - - public Player(String id, Integer coins, Integer goods) { - this.id = id; - this.coins = coins; - this.goods = goods; - } - - public Player() { - super(); - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public Integer getCoins() { - return coins; - } - - public void setCoins(Integer coins) { - this.coins = coins; - } - - public Integer getGoods() { - return goods; - } - - public void setGoods(Integer goods) { - this.goods = goods; - } -} -``` - -`PlayerMapper.java` is a mapping interface file generated using Mybatis Generator. This file only defines the interface, and the implementation classes of interface are automatically generated using XML or annotations. - -```java -package com.pingcap.model; - -import com.pingcap.model.Player; - -public interface PlayerMapper { - int deleteByPrimaryKey(String id); - - int insert(Player row); - - int insertSelective(Player row); - - Player selectByPrimaryKey(String id); - - int updateByPrimaryKeySelective(Player row); - - int updateByPrimaryKey(Player row); -} -``` - -`PlayerMapper.xml` is a mapping XML file generated using Mybatis Generator. Mybatis uses this to automatically generate the implementation class of the `PlayerMapper` interface. - -```xml - - - - - - - - - - - - id, coins, goods - - - - delete from player - where id = #{id,jdbcType=VARCHAR} - - - insert into player (id, coins, goods - ) - values (#{id,jdbcType=VARCHAR}, #{coins,jdbcType=INTEGER}, #{goods,jdbcType=INTEGER} - ) - - - insert into player - - - id, - - - coins, - - - goods, - - - - - #{id,jdbcType=VARCHAR}, - - - #{coins,jdbcType=INTEGER}, - - - #{goods,jdbcType=INTEGER}, - - - - - update player - - - coins = #{coins,jdbcType=INTEGER}, - - - goods = #{goods,jdbcType=INTEGER}, - - - where id = #{id,jdbcType=VARCHAR} - - - update player - set coins = #{coins,jdbcType=INTEGER}, - goods = #{goods,jdbcType=INTEGER} - where id = #{id,jdbcType=VARCHAR} - - -``` - -Since Mybatis Generator needs to generate the source code from the table definition, the table needs to be created first. To create the table, you can use `dbinit.sql`. - -```sql -USE test; -DROP TABLE IF EXISTS player; - -CREATE TABLE player ( - `id` VARCHAR(36), - `coins` INTEGER, - `goods` INTEGER, - PRIMARY KEY (`id`) -); -``` - -Split the interface `PlayerMapperEx` additionally to extend from `PlayerMapper` and write a matching `PlayerMapperEx.xml` file. Avoid changing `PlayerMapper.java` and `PlayerMapper.xml` directly. This is to avoid overwrite by Mybatis Generator. - -Define the added interface in `PlayerMapperEx.java`: - -```java -package com.pingcap.model; - -import java.util.List; - -public interface PlayerMapperEx extends PlayerMapper { - Player selectByPrimaryKeyWithLock(String id); - - List selectByLimit(Integer limit); - - Integer count(); -} -``` - -Define the mapping rules in `PlayerMapperEx.xml`: - -```xml - - - - - - - - - - - - id, coins, goods - - - - - - - - - -``` - -`PlayerDAO.java` is a class used to manage data, in which `DAO` means [Data Access Object](https://en.wikipedia.org/wiki/Data_access_object). The class defines a set of data manipulation methods for writing data. In it, Mybatis encapsulates a large number of operations such as object mapping and CRUD of basic objects, which greatly simplifies the code. - -```java -package com.pingcap.dao; - -import com.pingcap.model.Player; -import com.pingcap.model.PlayerMapperEx; -import org.apache.ibatis.session.SqlSession; -import org.apache.ibatis.session.SqlSessionFactory; - -import java.util.List; -import java.util.function.Function; - -public class PlayerDAO { - public static class NotEnoughException extends RuntimeException { - public NotEnoughException(String message) { - super(message); - } - } - - // Run SQL code in a way that automatically handles the - // transaction retry logic, so we don't have to duplicate it in - // various places. - public Object runTransaction(SqlSessionFactory sessionFactory, Function fn) { - Object resultObject = null; - SqlSession session = null; - - try { - // open a session with autoCommit is false - session = sessionFactory.openSession(false); - - // get player mapper - PlayerMapperEx playerMapperEx = session.getMapper(PlayerMapperEx.class); - - resultObject = fn.apply(playerMapperEx); - session.commit(); - System.out.println("APP: COMMIT;"); - } catch (Exception e) { - if (e instanceof NotEnoughException) { - System.out.printf("APP: ROLLBACK BY LOGIC; \n%s\n", e.getMessage()); - } else { - System.out.printf("APP: ROLLBACK BY ERROR; \n%s\n", e.getMessage()); - } - - if (session != null) { - session.rollback(); - } - } finally { - if (session != null) { - session.close(); - } - } - - return resultObject; - } - - public Function createPlayers(List players) { - return playerMapperEx -> { - Integer addedPlayerAmount = 0; - for (Player player: players) { - playerMapperEx.insert(player); - addedPlayerAmount ++; - } - System.out.printf("APP: createPlayers() --> %d\n", addedPlayerAmount); - return addedPlayerAmount; - }; - } - - public Function buyGoods(String sellId, String buyId, Integer amount, Integer price) { - return playerMapperEx -> { - Player sellPlayer = playerMapperEx.selectByPrimaryKeyWithLock(sellId); - Player buyPlayer = playerMapperEx.selectByPrimaryKeyWithLock(buyId); - - if (buyPlayer == null || sellPlayer == null) { - throw new NotEnoughException("sell or buy player not exist"); - } - - if (buyPlayer.getCoins() < price || sellPlayer.getGoods() < amount) { - throw new NotEnoughException("coins or goods not enough, rollback"); - } - - int affectRows = 0; - buyPlayer.setGoods(buyPlayer.getGoods() + amount); - buyPlayer.setCoins(buyPlayer.getCoins() - price); - affectRows += playerMapperEx.updateByPrimaryKey(buyPlayer); - - sellPlayer.setGoods(sellPlayer.getGoods() - amount); - sellPlayer.setCoins(sellPlayer.getCoins() + price); - affectRows += playerMapperEx.updateByPrimaryKey(sellPlayer); - - System.out.printf("APP: buyGoods --> sell: %s, buy: %s, amount: %d, price: %d\n", sellId, buyId, amount, price); - return affectRows; - }; - } - - public Function getPlayerByID(String id) { - return playerMapperEx -> playerMapperEx.selectByPrimaryKey(id); - } - - public Function printPlayers(Integer limit) { - return playerMapperEx -> { - List players = playerMapperEx.selectByLimit(limit); - - for (Player player: players) { - System.out.println("\n[printPlayers]:\n" + player); - } - return 0; - }; - } - - public Function countPlayers() { - return PlayerMapperEx::count; - } -} -``` - -`MybatisExample` is the main class of the `plain-java-mybatis` sample application. It defines the entry functions: - -```java -package com.pingcap; - -import com.pingcap.dao.PlayerDAO; -import com.pingcap.model.Player; -import org.apache.ibatis.io.Resources; -import org.apache.ibatis.session.SqlSessionFactory; -import org.apache.ibatis.session.SqlSessionFactoryBuilder; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; -import java.util.Collections; - -public class MybatisExample { - public static void main( String[] args ) throws IOException { - // 1. Create a SqlSessionFactory based on our mybatis-config.xml configuration - // file, which defines how to connect to the database. - InputStream inputStream = Resources.getResourceAsStream("mybatis-config.xml"); - SqlSessionFactory sessionFactory = new SqlSessionFactoryBuilder().build(inputStream); - - // 2. And then, create DAO to manager your data - PlayerDAO playerDAO = new PlayerDAO(); - - // 3. Run some simple examples. - - // Create a player who has 1 coin and 1 goods. - playerDAO.runTransaction(sessionFactory, playerDAO.createPlayers( - Collections.singletonList(new Player("test", 1, 1)))); - - // Get a player. - Player testPlayer = (Player)playerDAO.runTransaction(sessionFactory, playerDAO.getPlayerByID("test")); - System.out.printf("PlayerDAO.getPlayer:\n => id: %s\n => coins: %s\n => goods: %s\n", - testPlayer.getId(), testPlayer.getCoins(), testPlayer.getGoods()); - - // Count players amount. - Integer count = (Integer)playerDAO.runTransaction(sessionFactory, playerDAO.countPlayers()); - System.out.printf("PlayerDAO.countPlayers:\n => %d total players\n", count); - - // Print 3 players. - playerDAO.runTransaction(sessionFactory, playerDAO.printPlayers(3)); - - // 4. Getting further. - - // Player 1: id is "1", has only 100 coins. - // Player 2: id is "2", has 114514 coins, and 20 goods. - Player player1 = new Player("1", 100, 0); - Player player2 = new Player("2", 114514, 20); - - // Create two players "by hand", using the INSERT statement on the backend. - int addedCount = (Integer)playerDAO.runTransaction(sessionFactory, - playerDAO.createPlayers(Arrays.asList(player1, player2))); - System.out.printf("PlayerDAO.createPlayers:\n => %d total inserted players\n", addedCount); - - // Player 1 wants to buy 10 goods from player 2. - // It will cost 500 coins, but player 1 cannot afford it. - System.out.println("\nPlayerDAO.buyGoods:\n => this trade will fail"); - Integer updatedCount = (Integer)playerDAO.runTransaction(sessionFactory, - playerDAO.buyGoods(player2.getId(), player1.getId(), 10, 500)); - System.out.printf("PlayerDAO.buyGoods:\n => %d total update players\n", updatedCount); - - // So player 1 has to reduce the incoming quantity to two. - System.out.println("\nPlayerDAO.buyGoods:\n => this trade will success"); - updatedCount = (Integer)playerDAO.runTransaction(sessionFactory, - playerDAO.buyGoods(player2.getId(), player1.getId(), 2, 100)); - System.out.printf("PlayerDAO.buyGoods:\n => %d total update players\n", updatedCount); - } -} -``` - - - -
    - -Compared with Hibernate, the JDBC implementation might be not a best practice, because you need to write error handling logic manually and cannot reuse code easily, which makes your code slightly redundant. - -Hibernate is a popular open-source Java ORM, and it supports TiDB dialect starting from `v6.0.0.Beta2`, which fits TiDB features well. The following instructions take `v6.0.0.Beta2` as an example. - -Change to the `plain-java-hibernate` directory: - -```shell -cd plain-java-hibernate -``` - -The structure of this directory is as follows: - -``` -. -├── Makefile -├── plain-java-hibernate.iml -├── pom.xml -└── src - └── main - ├── java - │ └── com - │ └── pingcap - │ └── HibernateExample.java - └── resources - └── hibernate.cfg.xml -``` - -`hibernate.cfg.xml` is the Hibernate configuration file: - -```xml - - - - - - - com.mysql.cj.jdbc.Driver - org.hibernate.dialect.TiDBDialect - jdbc:mysql://localhost:4000/test - root - - false - - - create-drop - - - true - true - - -``` - -`HibernateExample.java` is the main body of the `plain-java-hibernate`. Compared with JDBC, when using Hibernate, you only need to write the path of the configuration file, because Hibernate avoids differences in database creation between different databases. - -`PlayerDAO` is a class used to manage data, in which `DAO` means [Data Access Object](https://en.wikipedia.org/wiki/Data_access_object). The class defines a set of data manipulation methods for writing data. Compared with JDBC, Hibernate encapsulates a large number of operations such as object mapping and CRUD of basic objects, which greatly simplifies the code. - -`PlayerBean` is a data entity class that is a mapping for tables. Each property of a `PlayerBean` corresponds to a field in the `player` table. Compared with JDBC, `PlayerBean` in Hibernate adds annotations to indicate mapping relationships for more information. - -```java -package com.pingcap; - -import jakarta.persistence.Column; -import jakarta.persistence.Entity; -import jakarta.persistence.Id; -import jakarta.persistence.Table; -import org.hibernate.JDBCException; -import org.hibernate.Session; -import org.hibernate.SessionFactory; -import org.hibernate.Transaction; -import org.hibernate.cfg.Configuration; -import org.hibernate.query.NativeQuery; -import org.hibernate.query.Query; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.function.Function; - -@Entity -@Table(name = "player_hibernate") -class PlayerBean { - @Id - private String id; - @Column(name = "coins") - private Integer coins; - @Column(name = "goods") - private Integer goods; - - public PlayerBean() { - } - - public PlayerBean(String id, Integer coins, Integer goods) { - this.id = id; - this.coins = coins; - this.goods = goods; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public Integer getCoins() { - return coins; - } - - public void setCoins(Integer coins) { - this.coins = coins; - } - - public Integer getGoods() { - return goods; - } - - public void setGoods(Integer goods) { - this.goods = goods; - } - - @Override - public String toString() { - return String.format(" %-8s => %10s\n %-8s => %10s\n %-8s => %10s\n", - "id", this.id, "coins", this.coins, "goods", this.goods); - } -} - -/** - * Main class for the basic Hibernate example. - **/ -public class HibernateExample -{ - public static class PlayerDAO { - public static class NotEnoughException extends RuntimeException { - public NotEnoughException(String message) { - super(message); - } - } - - // Run SQL code in a way that automatically handles the - // transaction retry logic so we don't have to duplicate it in - // various places. - public Object runTransaction(Session session, Function fn) { - Object resultObject = null; - - Transaction txn = session.beginTransaction(); - try { - resultObject = fn.apply(session); - txn.commit(); - System.out.println("APP: COMMIT;"); - } catch (JDBCException e) { - System.out.println("APP: ROLLBACK BY JDBC ERROR;"); - txn.rollback(); - } catch (NotEnoughException e) { - System.out.printf("APP: ROLLBACK BY LOGIC; %s", e.getMessage()); - txn.rollback(); - } - return resultObject; - } - - public Function createPlayers(List players) throws JDBCException { - return session -> { - Integer addedPlayerAmount = 0; - for (PlayerBean player: players) { - session.persist(player); - addedPlayerAmount ++; - } - System.out.printf("APP: createPlayers() --> %d\n", addedPlayerAmount); - return addedPlayerAmount; - }; - } - - public Function buyGoods(String sellId, String buyId, Integer amount, Integer price) throws JDBCException { - return session -> { - PlayerBean sellPlayer = session.get(PlayerBean.class, sellId); - PlayerBean buyPlayer = session.get(PlayerBean.class, buyId); - - if (buyPlayer == null || sellPlayer == null) { - throw new NotEnoughException("sell or buy player not exist"); - } - - if (buyPlayer.getCoins() < price || sellPlayer.getGoods() < amount) { - throw new NotEnoughException("coins or goods not enough, rollback"); - } - - buyPlayer.setGoods(buyPlayer.getGoods() + amount); - buyPlayer.setCoins(buyPlayer.getCoins() - price); - session.persist(buyPlayer); - - sellPlayer.setGoods(sellPlayer.getGoods() - amount); - sellPlayer.setCoins(sellPlayer.getCoins() + price); - session.persist(sellPlayer); - - System.out.printf("APP: buyGoods --> sell: %s, buy: %s, amount: %d, price: %d\n", sellId, buyId, amount, price); - return 0; - }; - } - - public Function getPlayerByID(String id) throws JDBCException { - return session -> session.get(PlayerBean.class, id); - } - - public Function printPlayers(Integer limit) throws JDBCException { - return session -> { - NativeQuery limitQuery = session.createNativeQuery("SELECT * FROM player_hibernate LIMIT :limit", PlayerBean.class); - limitQuery.setParameter("limit", limit); - List players = limitQuery.getResultList(); - - for (PlayerBean player: players) { - System.out.println("\n[printPlayers]:\n" + player); - } - return 0; - }; - } - - public Function countPlayers() throws JDBCException { - return session -> { - Query countQuery = session.createQuery("SELECT count(player_hibernate) FROM PlayerBean player_hibernate", Long.class); - return countQuery.getSingleResult(); - }; - } - } - - public static void main(String[] args) { - // 1. Create a SessionFactory based on our hibernate.cfg.xml configuration - // file, which defines how to connect to the database. - SessionFactory sessionFactory - = new Configuration() - .configure("hibernate.cfg.xml") - .addAnnotatedClass(PlayerBean.class) - .buildSessionFactory(); - - try (Session session = sessionFactory.openSession()) { - // 2. And then, create DAO to manager your data. - PlayerDAO playerDAO = new PlayerDAO(); - - // 3. Run some simple example. - - // Create a player who has 1 coin and 1 goods. - playerDAO.runTransaction(session, playerDAO.createPlayers(Collections.singletonList( - new PlayerBean("test", 1, 1)))); - - // Get a player. - PlayerBean testPlayer = (PlayerBean)playerDAO.runTransaction(session, playerDAO.getPlayerByID("test")); - System.out.printf("PlayerDAO.getPlayer:\n => id: %s\n => coins: %s\n => goods: %s\n", - testPlayer.getId(), testPlayer.getCoins(), testPlayer.getGoods()); - - // Count players amount. - Long count = (Long)playerDAO.runTransaction(session, playerDAO.countPlayers()); - System.out.printf("PlayerDAO.countPlayers:\n => %d total players\n", count); - - // Print 3 players. - playerDAO.runTransaction(session, playerDAO.printPlayers(3)); - - // 4. Getting further. - - // Player 1: id is "1", has only 100 coins. - // Player 2: id is "2", has 114514 coins, and 20 goods. - PlayerBean player1 = new PlayerBean("1", 100, 0); - PlayerBean player2 = new PlayerBean("2", 114514, 20); - - // Create two players "by hand", using the INSERT statement on the backend. - int addedCount = (Integer)playerDAO.runTransaction(session, - playerDAO.createPlayers(Arrays.asList(player1, player2))); - System.out.printf("PlayerDAO.createPlayers:\n => %d total inserted players\n", addedCount); - - // Player 1 wants to buy 10 goods from player 2. - // It will cost 500 coins, but player 1 can't afford it. - System.out.println("\nPlayerDAO.buyGoods:\n => this trade will fail"); - Integer updatedCount = (Integer)playerDAO.runTransaction(session, - playerDAO.buyGoods(player2.getId(), player1.getId(), 10, 500)); - System.out.printf("PlayerDAO.buyGoods:\n => %d total update players\n", updatedCount); - - // So player 1 have to reduce his incoming quantity to two. - System.out.println("\nPlayerDAO.buyGoods:\n => this trade will success"); - updatedCount = (Integer)playerDAO.runTransaction(session, - playerDAO.buyGoods(player2.getId(), player1.getId(), 2, 100)); - System.out.printf("PlayerDAO.buyGoods:\n => %d total update players\n", updatedCount); - } finally { - sessionFactory.close(); - } - } -} -``` - -
    - -
    - -Change to the `plain-java-jdbc` directory: - -```shell -cd plain-java-jdbc -``` - -The structure of this directory is as follows: - -``` -. -├── Makefile -├── plain-java-jdbc.iml -├── pom.xml -└── src - └── main - ├── java - │ └── com - │ └── pingcap - │ └── JDBCExample.java - └── resources - └── dbinit.sql -``` - -You can find initialization statements for the table creation in `dbinit.sql`: - -```sql -USE test; -DROP TABLE IF EXISTS player; - -CREATE TABLE player ( - `id` VARCHAR(36), - `coins` INTEGER, - `goods` INTEGER, - PRIMARY KEY (`id`) -); -``` - -`JDBCExample.java` is the main body of the `plain-java-jdbc`. TiDB is highly compatible with the MySQL protocol, so you need to initialize a MySQL source instance `MysqlDataSource` to connect to TiDB. Then, you can initialize `PlayerDAO` for object management and use it to read, edit, add, and delete data. - -`PlayerDAO` is a class used to manage data, in which `DAO` means [Data Access Object](https://en.wikipedia.org/wiki/Data_access_object). The class defines a set of data manipulation methods to provide the ability to write data. - -`PlayerBean` is a data entity class that is a mapping for tables. Each property of a `PlayerBean` corresponds to a field in the `player` table. - -```java -package com.pingcap; - -import com.mysql.cj.jdbc.MysqlDataSource; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.*; - -/** - * Main class for the basic JDBC example. - **/ -public class JDBCExample -{ - public static class PlayerBean { - private String id; - private Integer coins; - private Integer goods; - - public PlayerBean() { - } - - public PlayerBean(String id, Integer coins, Integer goods) { - this.id = id; - this.coins = coins; - this.goods = goods; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public Integer getCoins() { - return coins; - } - - public void setCoins(Integer coins) { - this.coins = coins; - } - - public Integer getGoods() { - return goods; - } - - public void setGoods(Integer goods) { - this.goods = goods; - } - - @Override - public String toString() { - return String.format(" %-8s => %10s\n %-8s => %10s\n %-8s => %10s\n", - "id", this.id, "coins", this.coins, "goods", this.goods); - } - } - - /** - * Data access object used by 'ExampleDataSource'. - * Example for CURD and bulk insert. - */ - public static class PlayerDAO { - private final MysqlDataSource ds; - private final Random rand = new Random(); - - PlayerDAO(MysqlDataSource ds) { - this.ds = ds; - } - - /** - * Create players by passing in a List of PlayerBean. - * - * @param players Will create players list - * @return The number of create accounts - */ - public int createPlayers(List players){ - int rows = 0; - - Connection connection = null; - PreparedStatement preparedStatement = null; - try { - connection = ds.getConnection(); - preparedStatement = connection.prepareStatement("INSERT INTO player (id, coins, goods) VALUES (?, ?, ?)"); - } catch (SQLException e) { - System.out.printf("[createPlayers] ERROR: { state => %s, cause => %s, message => %s }\n", - e.getSQLState(), e.getCause(), e.getMessage()); - e.printStackTrace(); - - return -1; - } - - try { - for (PlayerBean player : players) { - preparedStatement.setString(1, player.getId()); - preparedStatement.setInt(2, player.getCoins()); - preparedStatement.setInt(3, player.getGoods()); - - preparedStatement.execute(); - rows += preparedStatement.getUpdateCount(); - } - } catch (SQLException e) { - System.out.printf("[createPlayers] ERROR: { state => %s, cause => %s, message => %s }\n", - e.getSQLState(), e.getCause(), e.getMessage()); - e.printStackTrace(); - } finally { - try { - connection.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } - - System.out.printf("\n[createPlayers]:\n '%s'\n", preparedStatement); - return rows; - } - - /** - * Buy goods and transfer funds between one player and another in one transaction. - * @param sellId Sell player id. - * @param buyId Buy player id. - * @param amount Goods amount, if sell player has not enough goods, the trade will break. - * @param price Price should pay, if buy player has not enough coins, the trade will break. - * - * @return The number of effected players. - */ - public int buyGoods(String sellId, String buyId, Integer amount, Integer price) { - int effectPlayers = 0; - - Connection connection = null; - try { - connection = ds.getConnection(); - } catch (SQLException e) { - System.out.printf("[buyGoods] ERROR: { state => %s, cause => %s, message => %s }\n", - e.getSQLState(), e.getCause(), e.getMessage()); - e.printStackTrace(); - return effectPlayers; - } - - try { - connection.setAutoCommit(false); - - PreparedStatement playerQuery = connection.prepareStatement("SELECT * FROM player WHERE id=? OR id=? FOR UPDATE"); - playerQuery.setString(1, sellId); - playerQuery.setString(2, buyId); - playerQuery.execute(); - - PlayerBean sellPlayer = null; - PlayerBean buyPlayer = null; - - ResultSet playerQueryResultSet = playerQuery.getResultSet(); - while (playerQueryResultSet.next()) { - PlayerBean player = new PlayerBean( - playerQueryResultSet.getString("id"), - playerQueryResultSet.getInt("coins"), - playerQueryResultSet.getInt("goods") - ); - - System.out.println("\n[buyGoods]:\n 'check goods and coins enough'"); - System.out.println(player); - - if (sellId.equals(player.getId())) { - sellPlayer = player; - } else { - buyPlayer = player; - } - } - - if (sellPlayer == null || buyPlayer == null) { - throw new SQLException("player not exist."); - } - - if (sellPlayer.getGoods().compareTo(amount) < 0) { - throw new SQLException(String.format("sell player %s goods not enough.", sellId)); - } - - if (buyPlayer.getCoins().compareTo(price) < 0) { - throw new SQLException(String.format("buy player %s coins not enough.", buyId)); - } - - PreparedStatement transfer = connection.prepareStatement("UPDATE player set goods = goods + ?, coins = coins + ? WHERE id=?"); - transfer.setInt(1, -amount); - transfer.setInt(2, price); - transfer.setString(3, sellId); - transfer.execute(); - effectPlayers += transfer.getUpdateCount(); - - transfer.setInt(1, amount); - transfer.setInt(2, -price); - transfer.setString(3, buyId); - transfer.execute(); - effectPlayers += transfer.getUpdateCount(); - - connection.commit(); - - System.out.println("\n[buyGoods]:\n 'trade success'"); - } catch (SQLException e) { - System.out.printf("[buyGoods] ERROR: { state => %s, cause => %s, message => %s }\n", - e.getSQLState(), e.getCause(), e.getMessage()); - - try { - System.out.println("[buyGoods] Rollback"); - - connection.rollback(); - } catch (SQLException ex) { - // do nothing - } - } finally { - try { - connection.close(); - } catch (SQLException e) { - // do nothing - } - } - - return effectPlayers; - } - - /** - * Get the player info by id. - * - * @param id Player id. - * @return The player of this id. - */ - public PlayerBean getPlayer(String id) { - PlayerBean player = null; - - try (Connection connection = ds.getConnection()) { - PreparedStatement preparedStatement = connection.prepareStatement("SELECT * FROM player WHERE id = ?"); - preparedStatement.setString(1, id); - preparedStatement.execute(); - - ResultSet res = preparedStatement.executeQuery(); - if(!res.next()) { - System.out.printf("No players in the table with id %s", id); - } else { - player = new PlayerBean(res.getString("id"), res.getInt("coins"), res.getInt("goods")); - } - } catch (SQLException e) { - System.out.printf("PlayerDAO.getPlayer ERROR: { state => %s, cause => %s, message => %s }\n", - e.getSQLState(), e.getCause(), e.getMessage()); - } - - return player; - } - - /** - * Insert randomized account data (id, coins, goods) using the JDBC fast path for - * bulk inserts. The fastest way to get data into TiDB is using the - * TiDB Lightning(https://docs.pingcap.com/tidb/stable/tidb-lightning-overview). - * However, if you must bulk insert from the application using INSERT SQL, the best - * option is the method shown here. It will require the following: - * - * Add `rewriteBatchedStatements=true` to your JDBC connection settings. - * Setting rewriteBatchedStatements to true now causes CallableStatements - * with batched arguments to be re-written in the form "CALL (...); CALL (...); ..." - * to send the batch in as few client/server round trips as possible. - * https://dev.mysql.com/doc/relnotes/connector-j/5.1/en/news-5-1-3.html - * - * You can see the `rewriteBatchedStatements` param effect logic at - * implement function: `com.mysql.cj.jdbc.StatementImpl.executeBatchUsingMultiQueries` - * - * @param total Add players amount. - * @param batchSize Bulk insert size for per batch. - * - * @return The number of new accounts inserted. - */ - public int bulkInsertRandomPlayers(Integer total, Integer batchSize) { - int totalNewPlayers = 0; - - try (Connection connection = ds.getConnection()) { - // We're managing the commit lifecycle ourselves, so we can - // control the size of our batch inserts. - connection.setAutoCommit(false); - - // In this example we are adding 500 rows to the database, - // but it could be any number. What's important is that - // the batch size is 128. - try (PreparedStatement pstmt = connection.prepareStatement("INSERT INTO player (id, coins, goods) VALUES (?, ?, ?)")) { - for (int i=0; i<=(total/batchSize);i++) { - for (int j=0; j %s row(s) updated in this batch\n", count.length); - } - connection.commit(); - } catch (SQLException e) { - System.out.printf("PlayerDAO.bulkInsertRandomPlayers ERROR: { state => %s, cause => %s, message => %s }\n", - e.getSQLState(), e.getCause(), e.getMessage()); - } - } catch (SQLException e) { - System.out.printf("PlayerDAO.bulkInsertRandomPlayers ERROR: { state => %s, cause => %s, message => %s }\n", - e.getSQLState(), e.getCause(), e.getMessage()); - } - return totalNewPlayers; - } - - - /** - * Print a subset of players from the data store by limit. - * - * @param limit Print max size. - */ - public void printPlayers(Integer limit) { - try (Connection connection = ds.getConnection()) { - PreparedStatement preparedStatement = connection.prepareStatement("SELECT * FROM player LIMIT ?"); - preparedStatement.setInt(1, limit); - preparedStatement.execute(); - - ResultSet res = preparedStatement.executeQuery(); - while (!res.next()) { - PlayerBean player = new PlayerBean(res.getString("id"), - res.getInt("coins"), res.getInt("goods")); - System.out.println("\n[printPlayers]:\n" + player); - } - } catch (SQLException e) { - System.out.printf("PlayerDAO.printPlayers ERROR: { state => %s, cause => %s, message => %s }\n", - e.getSQLState(), e.getCause(), e.getMessage()); - } - } - - - /** - * Count players from the data store. - * - * @return All players count - */ - public int countPlayers() { - int count = 0; - - try (Connection connection = ds.getConnection()) { - PreparedStatement preparedStatement = connection.prepareStatement("SELECT count(*) FROM player"); - preparedStatement.execute(); - - ResultSet res = preparedStatement.executeQuery(); - if(res.next()) { - count = res.getInt(1); - } - } catch (SQLException e) { - System.out.printf("PlayerDAO.countPlayers ERROR: { state => %s, cause => %s, message => %s }\n", - e.getSQLState(), e.getCause(), e.getMessage()); - } - - return count; - } - } - - public static void main(String[] args) { - // 1. Configure the example database connection. - - // 1.1 Create a mysql data source instance. - MysqlDataSource mysqlDataSource = new MysqlDataSource(); - - // 1.2 Set server name, port, database name, username and password. - mysqlDataSource.setServerName("localhost"); - mysqlDataSource.setPortNumber(4000); - mysqlDataSource.setDatabaseName("test"); - mysqlDataSource.setUser("root"); - mysqlDataSource.setPassword(""); - - // Or you can use jdbc string instead. - // mysqlDataSource.setURL("jdbc:mysql://{host}:{port}/test?user={user}&password={password}"); - - // 2. And then, create DAO to manager your data. - PlayerDAO dao = new PlayerDAO(mysqlDataSource); - - // 3. Run some simple example. - - // Create a player, has a coin and a goods. - dao.createPlayers(Collections.singletonList(new PlayerBean("test", 1, 1))); - - // Get a player. - PlayerBean testPlayer = dao.getPlayer("test"); - System.out.printf("PlayerDAO.getPlayer:\n => id: %s\n => coins: %s\n => goods: %s\n", - testPlayer.getId(), testPlayer.getCoins(), testPlayer.getGoods()); - - // Create players with bulk inserts, insert 1919 players totally, and per batch for 114 players. - int addedCount = dao.bulkInsertRandomPlayers(1919, 114); - System.out.printf("PlayerDAO.bulkInsertRandomPlayers:\n => %d total inserted players\n", addedCount); - - // Count players amount. - int count = dao.countPlayers(); - System.out.printf("PlayerDAO.countPlayers:\n => %d total players\n", count); - - // Print 3 players. - dao.printPlayers(3); - - // 4. Getting further. - - // Player 1: id is "1", has only 100 coins. - // Player 2: id is "2", has 114514 coins, and 20 goods. - PlayerBean player1 = new PlayerBean("1", 100, 0); - PlayerBean player2 = new PlayerBean("2", 114514, 20); - - // Create two players "by hand", using the INSERT statement on the backend. - addedCount = dao.createPlayers(Arrays.asList(player1, player2)); - System.out.printf("PlayerDAO.createPlayers:\n => %d total inserted players\n", addedCount); - - // Player 1 wants to buy 10 goods from player 2. - // It will cost 500 coins, but player 1 can't afford it. - System.out.println("\nPlayerDAO.buyGoods:\n => this trade will fail"); - int updatedCount = dao.buyGoods(player2.getId(), player1.getId(), 10, 500); - System.out.printf("PlayerDAO.buyGoods:\n => %d total update players\n", updatedCount); - - // So player 1 have to reduce his incoming quantity to two. - System.out.println("\nPlayerDAO.buyGoods:\n => this trade will success"); - updatedCount = dao.buyGoods(player2.getId(), player1.getId(), 2, 100); - System.out.printf("PlayerDAO.buyGoods:\n => %d total update players\n", updatedCount); - } -} -``` - -
    - - - -## Step 3. Run the code - -The following content introduces how to run the code step by step. - -### Step 3.1 Table initialization - - - -
    - -When using Mybatis, you need to initialize the database tables manually. If you are using a local cluster, and MySQL client has been installed locally, you can run it directly in the `plain-java-mybatis` directory: - -```shell -make prepare -``` - -Or you can execute the following command: - -```shell -mysql --host 127.0.0.1 --port 4000 -u root < src/main/resources/dbinit.sql -``` - -If you are using a non-local cluster or MySQL client has not been installed, connect to your cluster and run the statement in the `src/main/resources/dbinit.sql` file. - -
    - -
    - -No need to initialize tables manually. - -
    - -
    - - - -When using JDBC, you need to initialize the database tables manually. If you are using a local cluster, and MySQL client has been installed locally, you can run it directly in the `plain-java-jdbc` directory: - -```shell -make mysql -``` - -Or you can execute the following command: - -```shell -mysql --host 127.0.0.1 --port 4000 -u root - - - -When using JDBC, you need to connect to your cluster and run the statement in the `src/main/resources/dbinit.sql` file to initialize the database tables manually. - - - -
    - -
    - -### Step 3.2 Modify parameters for TiDB Cloud - - - -
    - -If you are using a TiDB Serverless cluster, modify the `dataSource.url`, `dataSource.username`, `dataSource.password` in `mybatis-config.xml`. - -```xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -``` - -Suppose that the password you set is `123456`, and the connection parameters you get from the cluster details page are the following: - -- Endpoint: `xxx.tidbcloud.com` -- Port: `4000` -- User: `2aEp24QWEDLqRFs.root` - -In this case, you can modify the parameters in `dataSource` node as follows: - -```xml - - - - - ... - - - - - - - - ... - - -``` - -
    - -
    - -If you are using a TiDB Serverless cluster, modify the `hibernate.connection.url`, `hibernate.connection.username`, `hibernate.connection.password` in `hibernate.cfg.xml`. - -```xml - - - - - - - com.mysql.cj.jdbc.Driver - org.hibernate.dialect.TiDBDialect - jdbc:mysql://localhost:4000/test - root - - false - - - create-drop - - - true - true - - -``` - -Suppose that the password you set is `123456`, and the connection parameters you get from the cluster details page are the following: - -- Endpoint: `xxx.tidbcloud.com` -- Port: `4000` -- User: `2aEp24QWEDLqRFs.root` - -In this case, you can modify the parameters as follows: - -```xml - - - - - - - com.mysql.cj.jdbc.Driver - org.hibernate.dialect.TiDBDialect - jdbc:mysql://xxx.tidbcloud.com:4000/test?sslMode=VERIFY_IDENTITY&enabledTLSProtocols=TLSv1.2,TLSv1.3 - 2aEp24QWEDLqRFs.root - 123456 - false - - - create-drop - - - true - true - - -``` - -
    - -
    - -If you are using a TiDB Serverless cluster, modify the parameters of the host, port, user, and password in `JDBCExample.java`: - -```java -mysqlDataSource.setServerName("localhost"); -mysqlDataSource.setPortNumber(4000); -mysqlDataSource.setDatabaseName("test"); -mysqlDataSource.setUser("root"); -mysqlDataSource.setPassword(""); -``` - -Suppose that the password you set is `123456`, and the connection parameters you get from the cluster details page are the following: - -- Endpoint: `xxx.tidbcloud.com` -- Port: `4000` -- User: `2aEp24QWEDLqRFs.root` - -In this case, you can modify the parameters as follows: - -```java -mysqlDataSource.setServerName("xxx.tidbcloud.com"); -mysqlDataSource.setPortNumber(4000); -mysqlDataSource.setDatabaseName("test"); -mysqlDataSource.setUser("2aEp24QWEDLqRFs.root"); -mysqlDataSource.setPassword("123456"); -mysqlDataSource.setSslMode(PropertyDefinitions.SslMode.VERIFY_IDENTITY.name()); -mysqlDataSource.setEnabledTLSProtocols("TLSv1.2,TLSv1.3"); -``` - -
    - -
    - -### Step 3.3 Run - - - -
    - -To run the code, you can run `make prepare`, `make gen`, `make build` and `make run` respectively: - -```shell -make prepare -# this command executes : -# - `mysql --host 127.0.0.1 --port 4000 -u root < src/main/resources/dbinit.sql` -# - `mysql --host 127.0.0.1 --port 4000 -u root -e "TRUNCATE test.player"` - -make gen -# this command executes : -# - `rm -f src/main/java/com/pingcap/model/Player.java` -# - `rm -f src/main/java/com/pingcap/model/PlayerMapper.java` -# - `rm -f src/main/resources/mapper/PlayerMapper.xml` -# - `mvn mybatis-generator:generate` - -make build # this command executes `mvn clean package` -make run # this command executes `java -jar target/plain-java-mybatis-0.0.1-jar-with-dependencies.jar` -``` - -Or you can use the native commands: - -```shell -mysql --host 127.0.0.1 --port 4000 -u root < src/main/resources/dbinit.sql -mysql --host 127.0.0.1 --port 4000 -u root -e "TRUNCATE test.player" -rm -f src/main/java/com/pingcap/model/Player.java -rm -f src/main/java/com/pingcap/model/PlayerMapper.java -rm -f src/main/resources/mapper/PlayerMapper.xml -mvn mybatis-generator:generate -mvn clean package -java -jar target/plain-java-mybatis-0.0.1-jar-with-dependencies.jar -``` - -Or run the `make` command directly, which is a combination of `make prepare`, `make gen`, `make build` and `make run`. - -
    - -
    - -To run the code, you can run `make build` and `make run` respectively: - -```shell -make build # this command executes `mvn clean package` -make run # this command executes `java -jar target/plain-java-jdbc-0.0.1-jar-with-dependencies.jar` -``` - -Or you can use the native commands: - -```shell -mvn clean package -java -jar target/plain-java-jdbc-0.0.1-jar-with-dependencies.jar -``` - -Or run the `make` command directly, which is a combination of `make build` and `make run`. - -
    - -
    - -To run the code, you can run `make build` and `make run` respectively: - -```shell -make build # this command executes `mvn clean package` -make run # this command executes `java -jar target/plain-java-jdbc-0.0.1-jar-with-dependencies.jar` -``` - -Or you can use the native commands: - -```shell -mvn clean package -java -jar target/plain-java-jdbc-0.0.1-jar-with-dependencies.jar -``` - -Or run the `make` command directly, which is a combination of `make build` and `make run`. - -
    - -
    - -## Step 4. Expected output - - - -
    - -[Mybatis Expected Output](https://github.com/pingcap-inc/tidb-example-java/blob/main/Expected-Output.md#plain-java-mybatis) - -
    - -
    - -[Hibernate Expected Output](https://github.com/pingcap-inc/tidb-example-java/blob/main/Expected-Output.md#plain-java-hibernate) - -
    - -
    - -[JDBC Expected Output](https://github.com/pingcap-inc/tidb-example-java/blob/main/Expected-Output.md#plain-java-jdbc) - -
    - -
    diff --git a/tidb-cloud/tidb-cloud-htap-quickstart.md b/tidb-cloud/tidb-cloud-htap-quickstart.md index 0427f7a97a0b6..e7628aba99b57 100644 --- a/tidb-cloud/tidb-cloud-htap-quickstart.md +++ b/tidb-cloud/tidb-cloud-htap-quickstart.md @@ -12,7 +12,7 @@ This tutorial guides you through an easy way to experience the Hybrid Transactio ## Before you begin -Before experiencing the HTAP feature, follow [TiDB Cloud Quick Start](/tidb-cloud/tidb-cloud-quickstart.md) to create a cluster with TiFlash nodes, connect to the TiDB cluster, and import the Capital Bikeshare sample data to the cluster. +Before experiencing the HTAP feature, follow [TiDB Cloud Quick Start](/tidb-cloud/tidb-cloud-quickstart.md) to create a TiDB Cloud Serverless cluster and import the **Steam Game Stats** sample dataset to the cluster. ## Steps @@ -20,28 +20,28 @@ Before experiencing the HTAP feature, follow [TiDB Cloud Quick Start](/tidb-clou After a cluster with TiFlash nodes is created, TiKV does not replicate data to TiFlash by default. You need to execute DDL statements in a MySQL client of TiDB to specify the tables to be replicated. After that, TiDB will create the specified table replicas in TiFlash accordingly. -For example, to replicate the `trips` table (in the Capital Bikeshare sample data) to TiFlash, execute the following statements: +For example, to replicate the `games` table (in the **Steam Game Stats** sample dataset) to TiFlash, execute the following statements: ```sql -USE bikeshare; +USE game; ``` ```sql -ALTER TABLE trips SET TIFLASH REPLICA 1; +ALTER TABLE games SET TIFLASH REPLICA 2; ``` To check the replication progress, execute the following statement: ```sql -SELECT * FROM information_schema.tiflash_replica WHERE TABLE_SCHEMA = 'bikeshare' and TABLE_NAME = 'trips'; +SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_ID, REPLICA_COUNT, LOCATION_LABELS, AVAILABLE, PROGRESS FROM information_schema.tiflash_replica WHERE TABLE_SCHEMA = 'game' and TABLE_NAME = 'games'; ``` ```sql -+--------------+------------+----------+---------------+-----------------+-----------+----------+------------+ -| TABLE_SCHEMA | TABLE_NAME | TABLE_ID | REPLICA_COUNT | LOCATION_LABELS | AVAILABLE | PROGRESS | TABLE_MODE | -+--------------+------------+----------+---------------+-----------------+-----------+----------+------------+ -| bikeshare | trips | 88 | 1 | | 1 | 1 | NORMAL | -+--------------+------------+----------+---------------+-----------------+-----------+----------+------------+ ++--------------+------------+----------+---------------+-----------------+-----------+----------+ +| TABLE_SCHEMA | TABLE_NAME | TABLE_ID | REPLICA_COUNT | LOCATION_LABELS | AVAILABLE | PROGRESS | ++--------------+------------+----------+---------------+-----------------+-----------+----------+ +| game | games | 88 | 2 | | 1 | 1 | ++--------------+------------+----------+---------------+-----------------+-----------+----------+ 1 row in set (0.20 sec) ``` @@ -54,12 +54,20 @@ In the result of the preceding statement: When the process of replication is completed, you can start to run some queries. -For example, you can check the number of trips by different start and end stations: +For example, you can check the number of games released every year, as well as the average price and average playtime: ```sql -SELECT start_station_name, end_station_name, COUNT(ride_id) as count from `trips` -GROUP BY start_station_name, end_station_name -ORDER BY count ASC; +SELECT + YEAR(`release_date`) AS `release_year`, + COUNT(*) AS `games_released`, + AVG(`price`) AS `average_price`, + AVG(`average_playtime_forever`) AS `average_playtime` +FROM + `games` +GROUP BY + `release_year` +ORDER BY + `release_year` DESC; ``` ### Step 3. Compare the query performance between row-based storage and columnar storage @@ -69,12 +77,20 @@ In this step, you can compare the execution statistics between TiKV (row-based s - To get the execution statistics of this query using TiKV, execute the following statement: ```sql - EXPLAIN ANALYZE SELECT /*+ READ_FROM_STORAGE(TIKV[trips]) */ start_station_name, end_station_name, COUNT(ride_id) as count from `trips` - GROUP BY start_station_name, end_station_name - ORDER BY count ASC; + EXPLAIN ANALYZE SELECT /*+ READ_FROM_STORAGE(TIKV[games]) */ + YEAR(`release_date`) AS `release_year`, + COUNT(*) AS `games_released`, + AVG(`price`) AS `average_price`, + AVG(`average_playtime_forever`) AS `average_playtime` + FROM + `games` + GROUP BY + `release_year` + ORDER BY + `release_year` DESC; ``` - For tables with TiFlash replicas, the TiDB optimizer automatically determines whether to use either TiKV or TiFlash replicas based on the cost estimation. In the preceding `EXPLAIN ANALYZE` statement, `HINT /*+ READ_FROM_STORAGE(TIKV[trips]) */` is used to force the optimizer to choose TiKV so you can check the execution statistics of TiKV. + For tables with TiFlash replicas, the TiDB optimizer automatically determines whether to use either TiKV or TiFlash replicas based on the cost estimation. In the preceding `EXPLAIN ANALYZE` statement, the `/*+ READ_FROM_STORAGE(TIKV[games]) */` hint is used to force the optimizer to choose TiKV so you can check the execution statistics of TiKV. > **Note:** > @@ -83,41 +99,50 @@ In this step, you can compare the execution statistics between TiKV (row-based s In the output, you can get the execution time from the `execution info` column. ```sql - id | estRows | actRows | task | access object | execution info | operator info | memory | disk - ---------------------------+-----------+---------+-----------+---------------+-------------------------------------------+-----------------------------------------------+---------+--------- - Sort_5 | 633.00 | 73633 | root | | time:1.62s, loops:73 | Column#15 | 6.88 MB | 0 Bytes - └─Projection_7 | 633.00 | 73633 | root | | time:1.57s, loops:76, Concurrency:OFF... | bikeshare.trips.start_station_name... | 6.20 MB | N/A | 6.20 MB | N/A - └─HashAgg_15 | 633.00 | 73633 | root | | time:1.57s, loops:76, partial_worker:... | group by:bikeshare.trips.end_station_name... | 58.0 MB | N/A - └─TableReader_16 | 633.00 | 111679 | root | | time:1.34s, loops:3, cop_task: {num: ... | data:HashAgg_8 | 7.55 MB | N/A - └─HashAgg_8 | 633.00 | 111679 | cop[tikv] | | tikv_task:{proc max:830ms, min:470ms,... | group by:bikeshare.trips.end_station_name... | N/A | N/A - └─TableFullScan_14 | 816090.00 | 816090 | cop[tikv] | table:trips | tikv_task:{proc max:490ms, min:310ms,... | keep order:false | N/A | N/A + id | estRows | actRows | task | access object | execution info | operator info | memory | disk + ---------------------------+----------+---------+-----------+---------------+--------------------------------------------+-----------------------------------------------+---------+--------- + Sort_5 | 4019.00 | 28 | root | | time:672.7ms, loops:2, RU:1159.679690 | Column#36:desc | 18.0 KB | 0 Bytes + └─Projection_7 | 4019.00 | 28 | root | | time:672.7ms, loops:6, Concurrency:5 | year(game.games.release_date)->Column#36, ... | 35.5 KB | N/A + └─HashAgg_15 | 4019.00 | 28 | root | | time:672.6ms, loops:6, partial_worker:... | group by:Column#38, funcs:count(Column#39)... | 56.7 KB | N/A + └─TableReader_16 | 4019.00 | 28 | root | | time:672.4ms, loops:2, cop_task: {num:... | data:HashAgg_9 | 3.60 KB | N/A + └─HashAgg_9 | 4019.00 | 28 | cop[tikv] | | tikv_task:{proc max:300ms, min:0s, avg... | group by:year(game.games.release_date), ... | N/A | N/A + └─TableFullScan_14 | 68223.00 | 68223 | cop[tikv] | table:games | tikv_task:{proc max:290ms, min:0s, avg... | keep order:false | N/A | N/A (6 rows) ``` - To get the execution statistics of this query using TiFlash, execute the following statement: ```sql - EXPLAIN ANALYZE SELECT start_station_name, end_station_name, COUNT(ride_id) as count from `trips` - GROUP BY start_station_name, end_station_name - ORDER BY count ASC; + EXPLAIN ANALYZE SELECT + YEAR(`release_date`) AS `release_year`, + COUNT(*) AS `games_released`, + AVG(`price`) AS `average_price`, + AVG(`average_playtime_forever`) AS `average_playtime` + FROM + `games` + GROUP BY + `release_year` + ORDER BY + `release_year` DESC; ``` In the output, you can get the execution time from the `execution info` column. ```sql - id | estRows | actRows | task | access object | execution info | operator info | memory | disk - -----------------------------------+-----------+---------+--------------+---------------+-------------------------------------------+------------------------------------+---------+--------- - Sort_5 | 633.00 | 73633 | root | | time:420.2ms, loops:73 | Column#15 | 5.61 MB | 0 Bytes - └─Projection_7 | 633.00 | 73633 | root | | time:368.7ms, loops:73, Concurrency:OFF | bikeshare.trips.start_station_... | 4.94 MB | N/A - └─TableReader_34 | 633.00 | 73633 | root | | time:368.6ms, loops:73, cop_task: {num... | data:ExchangeSender_33 | N/A | N/A - └─ExchangeSender_33 | 633.00 | 73633 | mpp[tiflash] | | tiflash_task:{time:360.7ms, loops:1,... | ExchangeType: PassThrough | N/A | N/A - └─Projection_29 | 633.00 | 73633 | mpp[tiflash] | | tiflash_task:{time:330.7ms, loops:1,... | Column#15, bikeshare.trips.star... | N/A | N/A - └─HashAgg_30 | 633.00 | 73633 | mpp[tiflash] | | tiflash_task:{time:330.7ms, loops:1,... | group by:bikeshare.trips.end_st... | N/A | N/A - └─ExchangeReceiver_32 | 633.00 | 73633 | mpp[tiflash] | | tiflash_task:{time:280.7ms, loops:12,... | | N/A | N/A - └─ExchangeSender_31 | 633.00 | 73633 | mpp[tiflash] | | tiflash_task:{time:272.3ms, loops:256,... | ExchangeType: HashPartition, Ha... | N/A | N/A - └─HashAgg_12 | 633.00 | 73633 | mpp[tiflash] | | tiflash_task:{time:252.3ms, loops:256,... | group by:bikeshare.trips.end_st... | N/A | N/A - └─TableFullScan_28 | 816090.00 | 816090 | mpp[tiflash] | table:trips | tiflash_task:{time:92.3ms, loops:16,... | keep order:false | N/A | N/A - (10 rows) + id | estRows | actRows | task | access object | execution info | operator info | memory | disk + -------------------------------------+----------+---------+--------------+---------------+-------------------------------------------------------+--------------------------------------------+---------+--------- + Sort_5 | 4019.00 | 28 | root | | time:222.2ms, loops:2, RU:25.609675 | Column#36:desc | 3.77 KB | 0 Bytes + └─TableReader_39 | 4019.00 | 28 | root | | time:222.1ms, loops:2, cop_task: {num: 2, max: 0s,... | MppVersion: 1, data:ExchangeSender_38 | 4.64 KB | N/A + └─ExchangeSender_38 | 4019.00 | 28 | mpp[tiflash] | | tiflash_task:{time:214.8ms, loops:1, threads:1} | ExchangeType: PassThrough | N/A | N/A + └─Projection_8 | 4019.00 | 28 | mpp[tiflash] | | tiflash_task:{time:214.8ms, loops:1, threads:1} | year(game.games.release_date)->Column#3... | N/A | N/A + └─Projection_34 | 4019.00 | 28 | mpp[tiflash] | | tiflash_task:{time:214.8ms, loops:1, threads:1} | Column#33, div(Column#34, cast(case(eq(... | N/A | N/A + └─HashAgg_35 | 4019.00 | 28 | mpp[tiflash] | | tiflash_task:{time:214.8ms, loops:1, threads:1} | group by:Column#63, funcs:sum(Column#64... | N/A | N/A + └─ExchangeReceiver_37 | 4019.00 | 28 | mpp[tiflash] | | tiflash_task:{time:214.8ms, loops:1, threads:8} | | N/A | N/A + └─ExchangeSender_36 | 4019.00 | 28 | mpp[tiflash] | | tiflash_task:{time:210.6ms, loops:1, threads:1} | ExchangeType: HashPartition, Compressio... | N/A | N/A + └─HashAgg_33 | 4019.00 | 28 | mpp[tiflash] | | tiflash_task:{time:210.6ms, loops:1, threads:1} | group by:Column#75, funcs:count(1)->Col... | N/A | N/A + └─Projection_40 | 68223.00 | 68223 | mpp[tiflash] | | tiflash_task:{time:210.6ms, loops:2, threads:8} | game.games.price, game.games.price, gam... | N/A | N/A + └─TableFullScan_23 | 68223.00 | 68223 | mpp[tiflash] | table:games | tiflash_task:{time:210.6ms, loops:2, threads:8}, ... | keep order:false | N/A | N/A + (11 rows) ``` > **Note:** diff --git a/tidb-cloud/tidb-cloud-import-local-files.md b/tidb-cloud/tidb-cloud-import-local-files.md index 4c1f1f408d1fe..a92875192d8f2 100644 --- a/tidb-cloud/tidb-cloud-import-local-files.md +++ b/tidb-cloud/tidb-cloud-import-local-files.md @@ -12,13 +12,13 @@ Currently, this method supports importing one CSV file for one task into either ## Limitations - Currently, TiDB Cloud only supports importing a local file in CSV format within 50 MiB for one task. -- Importing local files is supported only for TiDB Serverless clusters, not for TiDB Dedicated clusters. +- Importing local files is supported only for TiDB Cloud Serverless clusters, not for TiDB Cloud Dedicated clusters. - You cannot run more than one import task at the same time. - When you import a CSV file into an existing table in TiDB Cloud and the target table has more columns than the source file, the extra columns are handled differently depending on the situation: - If the extra columns are not the primary keys or the unique keys, no error will be reported. Instead, these extra columns will be populated with their [default values](/data-type-default-values.md). - If the extra columns are the primary keys or the unique keys and do not have the `auto_increment` or `auto_random` attribute, an error will be reported. In that case, it is recommended that you choose one of the following strategies: - Provide a source file that includes these the primary keys or the unique keys columns. - - Set the attributes of the the primary key or the unique key columns to `auto_increment` or `auto_random`. + - Set the attributes of the primary key or the unique key columns to `auto_increment` or `auto_random`. - If a column name is a reserved [keyword](/keywords.md) in TiDB, TiDB Cloud automatically adds backticks `` ` `` to enclose the column name. For example, if the column name is `order`, TiDB Cloud automatically adds backticks `` ` `` to change it to `` `order` `` and imports the data into the target table. ## Import local files @@ -33,9 +33,9 @@ Currently, this method supports importing one CSV file for one task into either 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. -2. On the **Import** page, you can directly drag and drop your local file to the upload area, or click the upload area to select and upload the target local file. Note that you can upload only one CSV file of less than 50 MiB for one task. +2. On the **Import** page, you can directly drag and drop your local file to the upload area, or click **Upload a local file** to select and upload the target local file. Note that you can upload only one CSV file of less than 50 MiB for one task. If your local file is larger than 50 MiB, see [How to import a local file larger than 50 MiB?](#how-to-import-a-local-file-larger-than-50-mib). -3. In the **Target** area, select the target database and the target table, or enter a name directly to create a new database or a new table. The name must start with letters (a-z and A-Z) or numbers (0-9), and can contain letters (a-z and A-Z), numbers (0-9), and the underscore (_) character. Click **Preview**. +3. In the **Destination** section, select the target database and the target table, or enter a name directly to create a new database or a new table. The name must start with letters (a-z and A-Z) or numbers (0-9), and can contain letters (a-z and A-Z), numbers (0-9), and the underscore (_) character. Click **Define Table**, the **Table Definition** section is displayed. 4. Check the table. @@ -72,13 +72,13 @@ Currently, this method supports importing one CSV file for one task into either You can also click **Edit CSV configuration** to configure Backslash Escape, Separator, and Delimiter for more fine-grained control. For more information about the CSV configuration, see [CSV Configurations for Importing Data](/tidb-cloud/csv-config-for-import-data.md). -8. On the **Preview** page, you can have a preview of the data. Click **Start Import**. +8. Click **Start Import**. You can view the import progress on the **Import Task Detail** page. If there are warnings or failed tasks, you can check to view the details and solve them. -9. After the import task is completed, you can click **Explore your data by Chat2Query** to query your imported data. For more information about how to use Chat2Qury, see [Explore Your Data with AI-Powered Chat2Query](/tidb-cloud/explore-data-with-chat2query.md). +9. After the import task is completed, you can click **Explore your data by SQL Editor** to query your imported data. For more information about how to use SQL Editor, see [Explore your data with AI-assisted SQL Editor](/tidb-cloud/explore-data-with-chat2query.md). -10. On the **Import** page, you can click **View** in the **Action** column to check the import task detail. +10. On the **Import** page, you can click **...** > **View** in the **Action** column to check the import task detail. ## FAQ @@ -103,3 +103,35 @@ If you use `mysql` and encounter `ERROR 2068 (HY000): LOAD DATA LOCAL INFILE fil ### Why can't I query a column with a reserved keyword after importing data into TiDB Cloud? If a column name is a reserved [keyword](/keywords.md) in TiDB, TiDB Cloud automatically adds backticks `` ` `` to enclose the column name and then imports the data into the target table. When you query the column, you need to add backticks `` ` `` to enclose the column name. For example, if the column name is `order`, you need to query the column with `` `order` ``. + +### How to import a local file larger than 50 MiB? + +If the file is larger than 50 MiB, you can use the `split [-l ${line_count}]` utility to split it into multiple smaller files (for Linux or macOS only). For example, run `split -l 100000 tidb-01.csv small_files` to split a file named `tidb-01.csv` by line length `100000`, and the split files are named `small_files${suffix}`. Then, you can import these smaller files to TiDB Cloud one by one. + +Refer to the following script: + +```bash +#!/bin/bash +n=$1 +file_path=$2 +file_extension="${file_path##*.}" +file_name="${file_path%.*}" +total_lines=$(wc -l < $file_path) +lines_per_file=$(( (total_lines + n - 1) / n )) +split -d -a 1 -l $lines_per_file $file_path $file_name. +for (( i=0; i<$n; i++ )) +do + mv $file_name.$i $file_name.$i.$file_extension +done +``` + +You can input `n` and a file name, and then run the script. The script will divide the file into `n` equal parts while keeping the original file extension. For example: + +```bash +> sh ./split.sh 3 mytest.customer.csv +> ls -h | grep mytest +mytest.customer.0.csv +mytest.customer.1.csv +mytest.customer.2.csv +mytest.customer.csv +``` diff --git a/tidb-cloud/tidb-cloud-intro.md b/tidb-cloud/tidb-cloud-intro.md index 03a38a2e03bd9..e9fc126a9065f 100644 --- a/tidb-cloud/tidb-cloud-intro.md +++ b/tidb-cloud/tidb-cloud-intro.md @@ -1,10 +1,10 @@ --- -title: TiDB Cloud Introduction +title: What is TiDB Cloud summary: Learn about TiDB Cloud and its architecture. category: intro --- -# TiDB Cloud Introduction +# What is TiDB Cloud [TiDB Cloud](https://www.pingcap.com/tidb-cloud/) is a fully-managed Database-as-a-Service (DBaaS) that brings [TiDB](https://docs.pingcap.com/tidb/stable/overview), an open-source Hybrid Transactional and Analytical Processing (HTAP) database, to your cloud. TiDB Cloud offers an easy way to deploy and manage databases to let you focus on your applications, not the complexities of the databases. You can create TiDB Cloud clusters to quickly build mission-critical applications on Google Cloud and Amazon Web Services (AWS). @@ -64,15 +64,15 @@ With TiDB Cloud, you can get the following key features: TiDB Cloud provides the following two deployment options: -- [TiDB Serverless](https://www.pingcap.com/tidb-serverless) +- [TiDB Cloud Serverless](https://www.pingcap.com/tidb-serverless) - TiDB Serverless is a fully managed, multi-tenant TiDB offering. It delivers an instant, autoscaling MySQL-compatible database and offers a generous free tier and consumption based billing once free limits are exceeded. + TiDB Cloud Serverless is a fully managed, multi-tenant TiDB offering. It delivers an instant, autoscaling MySQL-compatible database and offers a generous free tier and consumption based billing once free limits are exceeded. -- [TiDB Dedicated](https://www.pingcap.com/tidb-dedicated) +- [TiDB Cloud Dedicated](https://www.pingcap.com/tidb-dedicated) - TiDB Dedicated is for production use with the benefits of cross-zone high availability, horizontal scaling, and [HTAP](https://en.wikipedia.org/wiki/Hybrid_transactional/analytical_processing). + TiDB Cloud Dedicated is for production use with the benefits of cross-zone high availability, horizontal scaling, and [HTAP](https://en.wikipedia.org/wiki/Hybrid_transactional/analytical_processing). -For feature comparison between TiDB Serverless and TiDB Dedicated, see [TiDB: An advanced, open source, distributed SQL database](https://www.pingcap.com/get-started-tidb). +For feature comparison between TiDB Cloud Serverless and TiDB Cloud Dedicated, see [TiDB: An advanced, open source, distributed SQL database](https://www.pingcap.com/get-started-tidb). ## Architecture @@ -80,7 +80,7 @@ For feature comparison between TiDB Serverless and TiDB Dedicated, see [TiDB: An - TiDB VPC (Virtual Private Cloud) - For each TiDB Cloud cluster, all TiDB nodes and auxiliary nodes, including TiDB Operator nodes and logging nodes, are deployed in an independent VPC. + For each TiDB Cloud cluster, all TiDB nodes and auxiliary nodes, including TiDB Operator nodes and logging nodes, are deployed in the same VPC. - TiDB Cloud Central Services diff --git a/tidb-cloud/tidb-cloud-migration-overview.md b/tidb-cloud/tidb-cloud-migration-overview.md index 9ff553e3ebe36..e6dfcd50e4bb3 100644 --- a/tidb-cloud/tidb-cloud-migration-overview.md +++ b/tidb-cloud/tidb-cloud-migration-overview.md @@ -27,9 +27,9 @@ When you migrate data from a MySQL-compatible database, you can perform full dat If your application uses MySQL shards for data storage, you can migrate these shards into TiDB Cloud as one table. For more information, see [Migrate and Merge MySQL Shards of Large Datasets to TiDB Cloud](/tidb-cloud/migrate-sql-shards.md). -- Migrate from TiDB Self-Hosted +- Migrate from TiDB Self-Managed - You can migrate data from your TiDB Self-Hosted clusters to TiDB Cloud (AWS) through Dumpling and TiCDC. For more information, see [Migrate from TiDB Self-Hosted to TiDB Cloud](/tidb-cloud/migrate-from-op-tidb.md). + You can migrate data from your TiDB Self-Managed clusters to TiDB Cloud (AWS) through Dumpling and TiCDC. For more information, see [Migrate from TiDB Self-Managed to TiDB Cloud](/tidb-cloud/migrate-from-op-tidb.md). ## Import data from files to TiDB Cloud diff --git a/tidb-cloud/tidb-cloud-org-sso-authentication.md b/tidb-cloud/tidb-cloud-org-sso-authentication.md index 8cf8e15465705..35526f55b5b02 100644 --- a/tidb-cloud/tidb-cloud-org-sso-authentication.md +++ b/tidb-cloud/tidb-cloud-org-sso-authentication.md @@ -19,8 +19,7 @@ In this document, you will learn how to migrate the authentication scheme of you > **Note:** > -> - The Cloud Organization SSO feature is in beta and is only available upon request. To request this feature, click **?** in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com) and click **Request Support**. Then, fill in "Apply for Cloud Organization SSO" in the **Description** field and click **Send**. -> - If your current TiDB login URL starts with `https://tidbcloud.com/enterprise/`, it means that Cloud Organization SSO is already enabled for your organization. +> The Cloud Organization SSO feature is only available for paid organizations. ## Before you begin @@ -43,13 +42,14 @@ The format of the custom URL is `https://tidbcloud.com/enterprise/signin/your-co TiDB Cloud provides the following authentication methods for Organization SSO. +- Username and password - Google - GitHub - Microsoft - OIDC - SAML -When you enable Cloud Organization SSO, the first three methods are enabled by default. +When you enable Cloud Organization SSO, the first four methods are enabled by default. If you want to enforce the use of SSO for your organization, you can disable the username and password authentication method. All the enabled authentication methods will be displayed on your custom TiDB Cloud login page, so you need to decide which authentication methods to be enabled or disabled in advance. @@ -77,7 +77,7 @@ To enable Cloud Organization SSO, take the following steps: 1. Log in to [TiDB Cloud console](https://tidbcloud.com) as a user with the `Organization Owner` role. 2. In the lower-left corner of the TiDB Cloud console, click , and then click **Organization Settings**. -3. On the **Organization Settings** page, click the **Authentication** tab, and then click **Enable**. +3. In the left navigation pane, click the **Authentication** tab, and then click **Enable**. 4. In the dialog, fill in the custom URL for your organization, which must be unique in TiDB Cloud. > **Note:** @@ -94,9 +94,9 @@ To enable Cloud Organization SSO, take the following steps: Enabling an authentication method in TiDB Cloud allows members using that method to log in to TiDB Cloud using your custom URL. -### Configure Google, GitHub, or Microsoft authentication methods +### Configure username and password, Google, GitHub, or Microsoft authentication methods -After enabling Cloud Organization Cloud, you can configure Google, GitHub, or Microsoft authentication methods as follows: +After enabling Cloud Organization Cloud, you can configure username and password, Google, GitHub, or Microsoft authentication methods as follows: 1. On the **Organization Settings** page, enable or disable the Google, GitHub, or Microsoft authentication methods according to your need. 2. For an enabled authentication method, you can click to configure the method details. @@ -157,6 +157,10 @@ In TiDB Cloud, the OIDC authentication method is disabled by default. After enab If you have an identity provider that uses the SAML identity protocol, you can enable the SAML authentication method for TiDB Cloud login. +> **Note:** +> +> TiDB Cloud uses email addresses as unique identifiers for different users. Therefore, ensure that the `email` attribute for your organization members is configured in your identity provider. + In TiDB Cloud, the SAML authentication method is disabled by default. After enabling Cloud Organization Cloud, you can enable and configure the SAML authentication method as follows: 1. Get the following information from your identity provider for TiDB Cloud Organization SSO: @@ -164,7 +168,7 @@ In TiDB Cloud, the SAML authentication method is disabled by default. After enab - Sign on URL - Signing Certificate -2. On the **Organization Settings** page, click the **Authentication** tab, locate the row of SAML in the **Authentication Methods** area, and then click to show the SAML method details. +2. On the **Organization Settings** page, click the **Authentication** tab in the left navigation pane, locate the row of SAML in the **Authentication Methods** area, and then click to show the SAML method details. 3. In the method details, you can configure the following: - **Name** @@ -191,4 +195,57 @@ In TiDB Cloud, the SAML authentication method is disabled by default. After enab > > If you have configured email domains, before saving the settings, make sure that you add the email domain that you currently use for login, to avoid that you are locked out by TiDB Cloud. -4. Click **Save**. \ No newline at end of file + - **SCIM Provisioning Accounts** + + It is disabled by default. You can enable it if you want to centralize and automate provisioning, deprovisioning, and identity management for TiDB Cloud organization users and groups from your identity provider. For detailed configuration steps, see [Configure SCIM provisioning](#configure-scim-provisioning). + +4. Click **Save**. + +#### Configure SCIM provisioning + +[System for Cross-domain Identity Management (SCIM)](https://www.rfc-editor.org/rfc/rfc7644) is an open standard that automates the exchange of user identity information between identity domains and IT systems. By configuring SCIM provisioning, user groups from your identity provider can be automatically synchronized to TiDB Cloud, and you can centrally manage roles for these groups in TiDB Cloud. + +> **Note:** +> +> SCIM provisioning can be enabled only on the [SAML authentication method](#configure-the-saml-authentication-method). + +1. In TiDB Cloud, enable the **SCIM Provisioning Accounts** option of the [SAML authentication method](#configure-the-saml-authentication-method), and then record the following information for later use. + + - SCIM connector base URL + - Unique identifier field for users + - Authentication Mode + +2. In your identity provider, configure SCIM provisioning for TiDB Cloud. + + 1. In your identity provider, add SCIM provisioning for your TiDB Cloud organization to your SAML app integration. + + For example, if your identity provider is Okta, see [Add SCIM provisioning to app integrations](https://help.okta.com/en-us/content/topics/apps/apps_app_integration_wizard_scim.htm). + + 2. Assign your SAML app integration to the desired groups in your identity provider so members in the groups can access and use the app integration. + + For example, if your identity provider is Okta, see [Assign an app integration to a group](https://help.okta.com/en-us/content/topics/provisioning/lcm/lcm-assign-app-groups.htm). + + 3. Push user groups from your identity provider to TiDB Cloud. + + For example, if your identity provider is Okta, see [Manage group push](https://help.okta.com/en-us/content/topics/users-groups-profiles/usgp-group-push-main.htm). + +3. In TiDB Cloud, view groups pushed from your identity provider. + + 1. In the lower-left corner of the [TiDB Cloud console](https://tidbcloud.com), click , and then click **Organization Settings**. + 2. In the left navigation pane, click the **Authentication** tab. + 3. Click the **Groups** tab. The groups synchronized from your identity provider are displayed. + 4. To view users in a group, click **View**. + +4. In TiDB Cloud, grant roles to the groups pushed from your identity provider. + + > **Note:** + > + > Granting a role to a group means all members in the group gain that role. If a group includes members already in your TiDB Cloud organization, these members also gain the new role of the group. + + 1. To grant organization roles to the groups, click **By organization**, and then configure the roles in the **Organization Role** column. To learn about permissions of organization roles, see [Organization roles](/tidb-cloud/manage-user-access.md#organization-roles). + 2. To grant project roles to the groups, click **By project**, and then configure the roles in the **Project Role** column. To learn about permissions of the project roles, see [Project roles](/tidb-cloud/manage-user-access.md#project-roles). + +5. If you change the members of the pushed groups in your identity provider, these changes are dynamically synchronized to the corresponding groups in TiDB Cloud. + + - If new members are added to the groups in your identity provider, these members gain the roles of the corresponding groups. + - If some members are removed from the groups in your identity provider, these members are also removed from the corresponding groups in TiDB Cloud. \ No newline at end of file diff --git a/tidb-cloud/tidb-cloud-partners.md b/tidb-cloud/tidb-cloud-partners.md new file mode 100644 index 0000000000000..fc0c442b520fc --- /dev/null +++ b/tidb-cloud/tidb-cloud-partners.md @@ -0,0 +1,72 @@ +--- +title: TiDB Cloud Partner Web Console +summary: Learn how to use the TiDB Cloud Partner web console as a reseller and Managed Service Provider (MSP). +aliases: ['/tidbcloud/managed-service-provider'] +--- + +# TiDB Cloud Partner Web Console + +TiDB Cloud Partner Web Console is designed for partners focused on SaaS solutions, with the goal of building and nurturing a strong partnership between PingCAP and our partners to better serve our customers. + +There are two types of TiDB Cloud partners: + +- Reseller: resells TiDB Cloud through AWS Marketplace Channel Partner Private Offer (CPPO) +- Managed Service Provider (MSP): resells TiDB Cloud and provides value-added services + +## Reseller through AWS Channel Partner Private Offer (CPPO) + +The reseller through [AWS CPPO](https://aws.amazon.com/marketplace/features/cpprivateoffers) allows customers to purchase TiDB Cloud through the AWS Marketplace directly from a reseller. This enables customers to benefit from the partner's business knowledge, localized support, and expertise, while still enjoying the fast and seamless purchasing experience they expect from AWS Marketplace. + +### Become a reseller of PingCAP + +If you are interested in the reseller program and would like to join as a partner, [contact sales](https://www.pingcap.com/partners/become-a-partner/) to enroll. + +### Manage daily tasks for a reseller + +As a reseller, you have two ways to manage your daily management tasks: + +- [TiDB Cloud Partner console](https://partner-console.tidbcloud.com) +- Partner Management API. You can find the open API documentation on the **Support** page of the TiDB Cloud Partner console. + +## Managed Service Provider (MSP) + +An MSP is a partner who resells TiDB Cloud and provides value-added services, including but not limited to TiDB Cloud organization management, billing services, and technical support. + +Benefits of becoming a managed service provider include: + +- Discounts and incentive programs +- Enablement training +- Increased visibility through certification +- Joint marketing opportunities + +### Become an MSP of PingCAP + +If you are interested in the MSP program and would like to join as a partner, [contact sales](https://www.pingcap.com/partners/become-a-partner/) to enroll. Please provide the following information: + +- Company name +- Company contact email +- Company official website URL +- Company logo (One SVG file for light mode and one SVG file for dark mode; a horizontal logo with 256 x 48 pixels is preferred) + +The preceding information is used to generate an exclusive sign-up URL and page with your company logo for your customers. + +We will carefully evaluate your request and get back to you soon. + +### Manage daily tasks for an MSP + +As a TiDB Cloud MSP partner, there are two methods for you to manage your daily management tasks: + +- [TiDB Cloud Partner console](https://partner-console.tidbcloud.com) +- [MSP Management API (deprecated)](https://docs.pingcap.com/tidbcloud/api/v1beta1/msp) + +After your complete the registration as a TiDB Cloud partner, you will receive an email notification to activate the account in the TiDB Cloud Partner console, and receive an API key for the MSP Management API. + +You can use the MSP management API to manage the following daily tasks: + +- Query the MSP monthly bill for a specific month +- Query credits applied to an MSP +- Query discounts applied to an MSP +- Query the monthly bill for a specific MSP customer +- Create a new sign-up URL for an MSP customer +- List all MSP customers +- Retrieve MSP customer information by the customer organization ID diff --git a/tidb-cloud/tidb-cloud-performance-reference.md b/tidb-cloud/tidb-cloud-performance-reference.md index 20761576bb498..d16663ebde40c 100644 --- a/tidb-cloud/tidb-cloud-performance-reference.md +++ b/tidb-cloud/tidb-cloud-performance-reference.md @@ -9,38 +9,25 @@ This document provides [Sysbench](https://github.com/akopytov/sysbench) performa > **Note:** > -> The tests are performed on TiDB v6.1.1, and the test results are based on the condition that the P95 latency is below 105 ms. - -In this document, the transaction models `Read Only`, `Read Write`, and `Write Only` represent read workloads, mixed workloads, and write workloads. - -## 2 vCPU performance - -Currently, the 2 vCPU support of TiDB and TiKV is still in beta. - -Test scales: - -- TiDB (2 vCPU, 8 GiB) \* 1; TiKV (2 vCPU, 8 GiB) \* 3 -- TiDB (2 vCPU, 8 GiB) \* 2; TiKV (2 vCPU, 8 GiB) \* 3 - -Test results: - -**TiDB (2 vCPU, 8 GiB) \* 1; TiKV (2 vCPU, 8 GiB) \* 3** - -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | -|-------------------|---------|----------|--------|----------------------|------------------| -| Read Only | 15 | 3,496.77 | 218.55 | 68.63 | 95.81 | -| Read Write | 5 | 1,545.90 | 77.29 | 64.68 | 94.10 | -| Write Only | 40 | 4,326.57 | 721.10 | 55.47 | 90.78 | - -**TiDB (2 vCPU, 8 GiB) \* 2; TiKV (2 vCPU, 8 GiB) \* 3** - -Test results: - -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | -|-------------------|---------|----------|----------|----------------------|------------------| -| Read Only | 30 | 7,584.08 | 474.00 | 63.29 | 99.33 | -| Read Write | 10 | 2,680.58 | 134.03 | 74.61 | 104.84 | -| Write Only | 80 | 7,618.77 | 1,269.79 | 63.00 | 97.55 | +> The tests are performed on TiDB v6.1.1, and the test results are based on the condition that the P95 transaction latency is below 105 ms. + +Here is an example of a Sysbench configuration file: + +```txt +mysql-host={TIDB_HOST} +mysql-port=4000 +mysql-user=root +mysql-password=password +mysql-db=sbtest +time=1200 +threads={100} +report-interval=10 +db-driver=mysql +mysql-ignore-errors=1062,2013,8028,9002,9007 +auto-inc=false +``` + +In this document, the transaction models `Read Only`, `Read Write`, and `Write Only` represent read workloads, mixed workloads, and write workloads. ## 4 vCPU performance @@ -53,7 +40,7 @@ Test results: **TiDB (4 vCPU, 16 GiB) \* 1; TiKV (4 vCPU, 16 GiB) \* 3** -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | |-------------------|---------|----------|----------|----------------------|------------------| | Read Only | 35 | 8,064.89 | 504.06 | 69.43 | 104.84 | | Read Write | 25 | 6,747.60 | 337.38 | 74.10 | 102.97 | @@ -61,7 +48,7 @@ Test results: **TiDB (4 vCPU, 16 GiB) \* 2; TiKV (4 vCPU, 16 GiB) \* 3** -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | |-------------------|---------|-----------|----------|----------------------|------------------| | Read Only | 65 | 16,805.76 | 1,050.36 | 61.88 | 95.81 | | Read Write | 45 | 12,940.36 | 647.02 | 69.55 | 99.33 | @@ -82,7 +69,7 @@ Test results: **TiDB (8 vCPU, 16 GiB) \* 2; TiKV (8 vCPU, 32 GiB) \* 3** -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | |-------------------|---------|-----------|----------|----------------------|------------------| | Read Only | 150 | 37,863.64 | 2,366.48 | 63.38 | 99.33 | | Read Write | 100 | 30,218.42 | 1,510.92 | 66.18 | 94.10 | @@ -90,7 +77,7 @@ Test results: **TiDB (8 vCPU, 16 GiB) \* 4; TiKV (8 vCPU, 32 GiB) \* 3** -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | |-------------------|---------|-----------|----------|----------------------|------------------| | Read Only | 300 | 74,190.40 | 4,636.90 | 64.69 | 104.84 | | Read Write | 200 | 53,351.84 | 2,667.59 | 74.97 | 97.55 | @@ -98,7 +85,7 @@ Test results: **TiDB (8 vCPU, 16 GiB) \* 4; TiKV (8 vCPU, 32 GiB) \* 6** -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | |-------------------|---------|-----------|-----------|----------------------|------------------| | Read Only | 300 | 75,713.04 | 4,732.06 | 63.39 | 102.97 | | Read Write | 200 | 62,640.62 | 3,132.03 | 63.85 | 95.81 | @@ -106,7 +93,7 @@ Test results: **TiDB (8 vCPU, 16 GiB) \* 6; TiKV (8 vCPU, 32 GiB) \* 9** -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | |-------------------|---------|------------|-----------|----------------------|------------------| | Read Only | 450 | 113,407.94 | 7,088.00 | 63.48 | 104.84 | | Read Write | 300 | 92,387.31 | 4,619.37 | 64.93 | 99.33 | @@ -114,7 +101,7 @@ Test results: **TiDB (8 vCPU, 16 GiB) \* 9; TiKV (8 vCPU, 32 GiB) \* 6** -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | |-------------------|---------|------------|-----------|----------------------|------------------| | Read Only | 650 | 168,486.65 | 10,530.42 | 61.72 | 101.13 | | Read Write | 400 | 106,853.63 | 5,342.68 | 74.86 | 101.13 | @@ -122,7 +109,7 @@ Test results: **TiDB (8 vCPU, 16 GiB) \* 12; TiKV (8 vCPU, 32 GiB) \* 9** -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | |-------------------|---------|------------|-----------|----------------------|------------------| | Read Only | 800 | 211,882.77 | 13,242.67 | 60.40 | 101.13 | | Read Write | 550 | 139,393.46 | 6,969.67 | 78.90 | 104.84 | @@ -139,7 +126,7 @@ Test results: **TiDB (16 vCPU, 32 GiB) \* 1; TiKV (16 vCPU, 64 GiB) \* 3** -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | |-------------------|---------|----------|---------|----------------------|------------------| | Read Only | 125 | 37448.41 | 2340.53 | 53.40 | 89.16 | | Read Write | 100 | 28903.99 | 1445.20 | 69.19 | 104.84 | @@ -147,8 +134,33 @@ Test results: **TiDB (16 vCPU, 32 GiB) \* 2; TiKV (16 vCPU, 64 GiB) \* 3** -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | |-------------------|---------|----------|----------|----------------------|------------------| | Read Only | 300 | 77238.30 | 4827.39 | 62.14 | 102.97 | | Read Write | 200 | 58241.15 | 2912.06 | 68.67 | 97.55 | | Write Only | 700 | 68829.89 | 11471.65 | 61.01 | 101.13 | + +## 32 vCPU performance + +Test scales: + +- TiDB (32 vCPU, 64 GiB) \* 1; TiKV (32 vCPU, 128 GiB) \* 3 +- TiDB (32 vCPU, 64 GiB) \* 2; TiKV (32 vCPU, 128 GiB) \* 3 + +Test results: + +**TiDB (32 vCPU, 64 GiB) \* 1; TiKV (32 vCPU, 128 GiB) \* 3** + +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | +|-------------------|---------|----------|---------|----------------------|------------------| +| Read Only | 300 | 83941.16 | 5246 | 57.20 | 87.6 | +| Read Write | 250 | 71290.31 | 3565 | 70.10 | 105.0 | +| Write Only | 700 | 72199.56 | 12033 | 58.20 | 101.0 | + +**TiDB (32 vCPU, 64 GiB) \* 2; TiKV (32 vCPU, 128 GiB) \* 3** + +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | +|-------------------|---------|-----------|----------|----------------------|------------------| +| Read Only | 650 | 163101.68 | 10194 | 63.8 | 99.3 | +| Read Write | 450 | 123152.74 | 6158 | 73.1 | 101 | +| Write Only | 1200 | 112333.16 | 18722 | 64.1 | 101 | diff --git a/tidb-cloud/tidb-cloud-poc.md b/tidb-cloud/tidb-cloud-poc.md index 841781e0ebc83..33e9a9bffbeae 100644 --- a/tidb-cloud/tidb-cloud-poc.md +++ b/tidb-cloud/tidb-cloud-poc.md @@ -13,7 +13,7 @@ This document describes the typical PoC procedures and aims to help you quickly If you are interested in doing a PoC, feel free to contact PingCAP before you get started. The support team can help you create a test plan and walk you through the PoC procedures smoothly. -Alternatively, you can [create a TiDB Serverless](/tidb-cloud/tidb-cloud-quickstart.md#step-1-create-a-tidb-cluster) to get familiar with TiDB Cloud for a quick evaluation. Note that the TiDB Serverless has some [special terms and conditions](/tidb-cloud/select-cluster-tier.md#tidb-serverless-special-terms-and-conditions). +Alternatively, you can [create a TiDB Cloud Serverless](/tidb-cloud/tidb-cloud-quickstart.md#step-1-create-a-tidb-cluster) to get familiar with TiDB Cloud for a quick evaluation. Note that the TiDB Cloud Serverless has some [special terms and conditions](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless-special-terms-and-conditions). ## Overview of the PoC procedures @@ -23,7 +23,7 @@ A typical TiDB Cloud PoC consists of the following steps: 1. Define success criteria and create a test plan 2. Identify characteristics of your workload -3. Sign up and create a TiDB Dedicated cluster for the PoC +3. Sign up and create a TiDB Cloud Dedicated cluster for the PoC 4. Adapt your schemas and SQL 5. Import data 6. Run your workload and evaluate results @@ -52,13 +52,13 @@ TiDB Cloud is suitable for various use cases that require high availability and - Horizontally scaling out or scaling in - Financial-grade high availability - Real-time HTAP -- Compatible with the MySQL 5.7 protocol and MySQL ecosystem +- Compatible with the MySQL protocol and MySQL ecosystem You might also be interested in using [TiFlash](https://docs.pingcap.com/tidb/stable/tiflash-overview), a columnar storage engine that helps speed up analytical processing. During the PoC, you can use the TiFlash feature at any time. -## Step 3. Sign up and create a TiDB Dedicated cluster for the PoC +## Step 3. Sign up and create a TiDB Cloud Dedicated cluster for the PoC -To create a [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) cluster for the PoC, take the following steps: +To create a [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) cluster for the PoC, take the following steps: 1. Fill in the PoC application form by doing one of the following: @@ -67,29 +67,29 @@ To create a [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) Once you submit the form, the TiDB Cloud Support team will review your application, contact you, and transfer credits to your account once the application is approved. You can also contact a PingCAP support engineer to assist with your PoC procedures to ensure the PoC runs as smoothly as possible. -2. Refer to [Create a TiDB Dedicated Cluster](/tidb-cloud/create-tidb-cluster.md) to create a TiDB Dedicated cluster for the PoC. +2. Refer to [Create a TiDB Cloud Dedicated Cluster](/tidb-cloud/create-tidb-cluster.md) to create a TiDB Cloud Dedicated cluster for the PoC. Capacity planning is recommended for cluster sizing before you create a cluster. You can start with estimated numbers of TiDB, TiKV, or TiFlash nodes, and scale out the cluster later to meet performance requirements. You can find more details in the following documents or consult our support team. - For more information about estimation practice, see [Size Your TiDB](/tidb-cloud/size-your-cluster.md). -- For configurations of the TiDB Dedicated cluster, see [Create a TiDB Dedicated Cluster](/tidb-cloud/create-tidb-cluster.md). Configure the cluster size for TiDB, TiKV, and TiFlash (optional) respectively. +- For configurations of the TiDB Cloud Dedicated cluster, see [Create a TiDB Cloud Dedicated Cluster](/tidb-cloud/create-tidb-cluster.md). Configure the cluster size for TiDB, TiKV, and TiFlash (optional) respectively. - For how to plan and optimize your PoC credits consumption effectively, see [FAQ](#faq) in this document. - For more information about scaling, see [Scale Your TiDB Cluster](/tidb-cloud/scale-tidb-cluster.md). -Once a dedicated PoC cluster is created, you are ready to load data and perform a series of tests. For how to connect to a TiDB cluster, see [Connect to Your TiDB Dedicated Cluster](/tidb-cloud/connect-to-tidb-cluster.md). +Once a dedicated PoC cluster is created, you are ready to load data and perform a series of tests. For how to connect to a TiDB cluster, see [Connect to Your TiDB Cloud Dedicated Cluster](/tidb-cloud/connect-to-tidb-cluster.md). For a newly created cluster, note the following configurations: - The default time zone (the **Create Time** column on the Dashboard) is UTC. You can change it to your local time zone by following [Set the Local Time Zone](/tidb-cloud/manage-user-access.md#set-the-time-zone-for-your-organization). -- The default backup setting on a new cluster is full database backup on a daily basis. You can specify a preferred backup time or back up data manually. For the default backup time and more details, see [Back up and Restore TiDB Cluster Data](/tidb-cloud/backup-and-restore.md#backup). +- The default backup setting on a new cluster is full database backup on a daily basis. You can specify a preferred backup time or back up data manually. For the default backup time and more details, see [Back up and Restore TiDB Cluster Data](/tidb-cloud/backup-and-restore.md#turn-on-auto-backup). ## Step 4. Adapt your schemas and SQL Next, you can load your database schemas to the TiDB cluster, including tables and indexes. -Because the amount of PoC credits is limited, to maximize the value of credits, it is recommended that you create a [TiDB Serverless cluster](/tidb-cloud/select-cluster-tier.md#tidb-serverless) for compatibility tests and preliminary analysis on TiDB Cloud. +Because the amount of PoC credits is limited, to maximize the value of credits, it is recommended that you create a [TiDB Cloud Serverless cluster](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) for compatibility tests and preliminary analysis on TiDB Cloud. -TiDB Cloud is highly compatible with MySQL 5.7. You can directly import your data into TiDB if it is MySQL-compatible or can be adapted to be compatible with MySQL. +TiDB Cloud is highly compatible with MySQL 8.0. You can directly import your data into TiDB if it is MySQL-compatible or can be adapted to be compatible with MySQL. For more information about compatibilities, see the following documents: @@ -130,7 +130,7 @@ You can import data in various formats to TiDB Cloud: Now you have created the environment, adapted the schemas, and imported data. It is time to test your workload. -Before testing the workload, consider performing a manual backup, so that you can restore the database to its original state if needed. For more information, see [Back up and Restore TiDB Cluster Data](/tidb-cloud/backup-and-restore.md#backup). +Before testing the workload, consider performing a manual backup, so that you can restore the database to its original state if needed. For more information, see [Back up and Restore TiDB Cluster Data](/tidb-cloud/backup-and-restore.md#turn-on-auto-backup). After kicking off the workload, you can observe the system using the following methods: @@ -175,13 +175,13 @@ Now the workload testing is finished, you can explore more features, for example - Backup - To avoid vendor lock-in, you can use daily full backup to migrate data to a new cluster and use [Dumpling](https://docs.pingcap.com/tidb/stable/dumpling-overview) to export data. For more information, see [Back Up and Restore TiDB Dedicated Data](/tidb-cloud/backup-and-restore.md#backup) and [Back Up and Restore TiDB Dedicated Data](/tidb-cloud/backup-and-restore-serverless.md#backup). + To avoid vendor lock-in, you can use daily full backup to migrate data to a new cluster and use [Dumpling](https://docs.pingcap.com/tidb/stable/dumpling-overview) to export data. For more information, see [Back Up and Restore TiDB Cloud Dedicated Data](/tidb-cloud/backup-and-restore.md#turn-on-auto-backup) and [Back Up and Restore TiDB Cloud Dedicated Data](/tidb-cloud/backup-and-restore-serverless.md#backup). ## Step 8. Clean up the environment and finish the PoC You have completed the full cycle of a PoC after you test TiDB Cloud using real workloads and get the testing results. These results help you determine if TiDB Cloud meets your expectations. Meanwhile, you have accumulated best practices for using TiDB Cloud. -If you want to try TiDB Cloud on a larger scale, for a new round of deployments and tests, such as deploying with other node storage sizes offered by TiDB Cloud, get full access to TiDB Cloud by creating a [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) cluster. +If you want to try TiDB Cloud on a larger scale, for a new round of deployments and tests, such as deploying with other node storage sizes offered by TiDB Cloud, get full access to TiDB Cloud by creating a [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) cluster. If your credits are running out and you want to continue with the PoC, contact the [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md) for consultation. diff --git a/tidb-cloud/tidb-cloud-quickstart.md b/tidb-cloud/tidb-cloud-quickstart.md index 7ffedf227e8eb..e0c6bdf6529e6 100644 --- a/tidb-cloud/tidb-cloud-quickstart.md +++ b/tidb-cloud/tidb-cloud-quickstart.md @@ -14,7 +14,7 @@ Additionally, you can try out TiDB features on [TiDB Playground](https://play.ti ## Step 1: Create a TiDB cluster -[TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) is the best way to get started with TiDB Cloud. To create a TiDB Serverless cluster, follow these steps: +[TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) is the best way to get started with TiDB Cloud. To create a TiDB Cloud Serverless cluster, follow these steps: 1. If you do not have a TiDB Cloud account, click [here](https://tidbcloud.com/free-trial) to sign up. @@ -24,25 +24,27 @@ Additionally, you can try out TiDB features on [TiDB Playground](https://play.ti The [**Clusters**](https://tidbcloud.com/console/clusters) page is displayed by default. -3. For new sign-up users, TiDB Cloud automatically creates a default TiDB Serverless cluster named `Cluster0` for you. +3. For new sign-up users, TiDB Cloud automatically creates a default TiDB Cloud Serverless cluster named `Cluster0` for you. - - To instantly try out TiDB Cloud features with this default cluster, proceed to [Step 2: Try AI-powered Chat2Query (beta)](#step-2-try-ai-powered-chat2query-beta). - - To create a new TiDB Serverless cluster on your own, follow these steps: + - To instantly try out TiDB Cloud features with this default cluster, proceed to [Step 2: Try AI-assisted SQL Editor](#step-2-try-ai-assisted-sql-editor). + - To create a new TiDB Cloud Serverless cluster on your own, follow these steps: 1. Click **Create Cluster**. - 2. On the **Create Cluster** page, **Serverless** is selected by default. Select the target region for your cluster, update the default cluster name if necessary, and then click **Create**. Your TiDB Serverless cluster will be created in approximately 30 seconds. + 2. On the **Create Cluster** page, **Serverless** is selected by default. Select the target region for your cluster, update the default cluster name if necessary, select your [cluster plan](/tidb-cloud/select-cluster-tier.md#cluster-plans), and then click **Create**. Your TiDB Cloud Serverless cluster will be created in approximately 30 seconds. -## Step 2: Try AI-powered Chat2Query (beta) +## Step 2: Try AI-assisted SQL Editor -TiDB Cloud is powered by AI. You can use Chat2Query (beta), an AI-powered SQL editor in the TiDB Cloud console, to maximize the value of your data. +You can use the built-in AI-assisted SQL Editor in the TiDB Cloud console to maximize your data value. This enables you to run SQL queries against databases without a local SQL client. You can intuitively view the query results in tables or charts and easily check the query logs. -In Chat2Query, you can either simply type `--` followed by your instructions to let AI automatically generate SQL queries, or write SQL queries manually and run them against databases without using a terminal. - -1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click on a cluster name to go to its overview page, and then click **Chat2Query** in the left navigation pane. +1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click on a cluster name to go to its overview page, and then click **SQL Editor** in the left navigation pane. 2. To try the AI capacity of TiDB Cloud, follow the on-screen instructions to allow PingCAP and OpenAI to use your code snippets for research and service improvement, and then click **Save and Get Started**. -3. In the editor, you can either simply type `--` followed by your instructions to let AI automatically generate SQL queries, or write SQL queries manually. +3. In SQL Editor, press + I on macOS (or Control + I on Windows or Linux) to instruct [Chat2Query (beta)](/tidb-cloud/tidb-cloud-glossary.md#chat2query) to generate SQL queries automatically. + + For example, to create a new table `test.t` with two columns (column `id` and column `name`), you can type `use test;` to specify the database, press + I, type `create a new table t with id and name` as the instruction, and then press **Enter** to let AI generate a SQL statement accordingly. + + For the generated statement, you can accept it by clicking **Accept** and then further edit it if needed, or reject it by clicking **Discard**. > **Note:** > @@ -76,7 +78,32 @@ In Chat2Query, you can either simply type `--` followed by your instructions to -After running the queries, you can immediately see the query logs and results at the bottom of the page. +After running the queries, you can immediately see the query logs and results at the bottom of the page. + +To let AI generate more SQL statements, you can type more instructions as shown in the following example: + +```sql +use test; + +-- create a new table t with id and name +CREATE TABLE + `t` (`id` INT, `name` VARCHAR(255)); + +-- add 3 rows +INSERT INTO + `t` (`id`, `name`) +VALUES + (1, 'row1'), + (2, 'row2'), + (3, 'row3'); + +-- query all +SELECT + `id`, + `name` +FROM + `t`; +``` ## Step 3: Try interactive tutorials @@ -84,12 +111,12 @@ TiDB Cloud offers interactive tutorials with carefully crafted sample datasets t 1. Click on the **?** icon in the lower-right corner of the console and select **Interactive Tutorials**. 2. In the tutorials list, select a tutorial card to start, such as **Steam Game Stats**. -3. Choose a TiDB Serverless cluster that you want to use for the tutorial, and click **Import Dataset**. The import process might take approximately one minute. +3. Choose a TiDB Cloud Serverless cluster that you want to use for the tutorial, and click **Import Dataset**. The import process might take approximately one minute. 4. Once the sample data is imported, follow the on-screen instructions to complete the tutorial. ## What's next -- To learn how to connect to your cluster using different methods, see [Connect to a TiDB Serverless cluster](/tidb-cloud/connect-to-tidb-cluster-serverless.md). -- For more information about how to use Chat2Query to explore your data, see [Chat2Query](/tidb-cloud/explore-data-with-chat2query.md). +- To learn how to connect to your cluster using different methods, see [Connect to a TiDB Cloud Serverless cluster](/tidb-cloud/connect-to-tidb-cluster-serverless.md). +- For more information about how to use SQL Editor and Chat2Query to explore your data, see [Explore your data with AI-assisted SQL Editor](/tidb-cloud/explore-data-with-chat2query.md). - For TiDB SQL usage, see [Explore SQL with TiDB](/basic-sql-operations.md). -- For production use with the benefits of cross-zone high availability, horizontal scaling, and [HTAP](https://en.wikipedia.org/wiki/Hybrid_transactional/analytical_processing), see [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). +- For production use with the benefits of cross-zone high availability, horizontal scaling, and [HTAP](https://en.wikipedia.org/wiki/Hybrid_transactional/analytical_processing), see [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). diff --git a/tidb-cloud/tidb-cloud-release-notes.md b/tidb-cloud/tidb-cloud-release-notes.md index 770f72074b78b..e08780259bfd3 100644 --- a/tidb-cloud/tidb-cloud-release-notes.md +++ b/tidb-cloud/tidb-cloud-release-notes.md @@ -1,822 +1,447 @@ --- -title: TiDB Cloud Release Notes in 2023 -summary: Learn about the release notes of TiDB Cloud in 2023. +title: TiDB Cloud Release Notes in 2024 +summary: Learn about the release notes of TiDB Cloud in 2024. aliases: ['/tidbcloud/supported-tidb-versions','/tidbcloud/release-notes'] --- -# TiDB Cloud Release Notes in 2023 +# TiDB Cloud Release Notes in 2024 -This page lists the release notes of [TiDB Cloud](https://www.pingcap.com/tidb-cloud/) in 2023. +This page lists the release notes of [TiDB Cloud](https://www.pingcap.com/tidb-cloud/) in 2024. -## August 23, 2023 +## November 26, 2024 **General changes** -- Support Google Cloud [Private Service Connect](https://cloud.google.com/vpc/docs/private-service-connect) for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. +- Upgrade the default TiDB version of new [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters from [v7.5.4](https://docs.pingcap.com/tidb/v7.5/release-7.5.4) to [v8.1.1](https://docs.pingcap.com/tidb/stable/release-8.1.1). - You can now create a private endpoint and establish a secure connection to a TiDB Dedicated cluster hosted on Google Cloud. +- [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) reduces costs for large data writes by up to 80% for the following scenarios: - Key benefits: + - When you perform write operations larger than 16 MiB in [autocommit mode](/transaction-overview.md#autocommit). + - When you perform write operations larger than 16 MiB in [optimistic transaction model](/optimistic-transaction.md). + - When you [import data into TiDB Cloud](/tidb-cloud/tidb-cloud-migration-overview.md#import-data-from-files-to-tidb-cloud). - - Intuitive operations: helps you create a private endpoint with only several steps. - - Enhanced security: establishes a secure connection to protect your data. - - Improved performance: provides low-latency and high-bandwidth connectivity. + This improvement enhances the efficiency and cost-effectiveness of your data operations, providing greater savings as your workload scales. - For more information, see [Connect via Private Endpoint with Google Cloud](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md). - -- Support using a changefeed to stream data from a [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) cluster to [Google Cloud Storage (GCS)](https://cloud.google.com/storage). - - You can now stream data from TiDB Cloud to GCS by using your own account's bucket and providing precisely tailored permissions. After replicating data to GCS, you can analyze the changes in your data as you wish. - - For more information, see [Sink to Cloud Storage](/tidb-cloud/changefeed-sink-to-cloud-storage.md). - -## August 15, 2023 +## November 19, 2024 **General changes** -- [Data Service (beta)](https://tidbcloud.com/console/data-service) supports pagination for `GET` requests to improve the development experience. - - For `GET` requests, you can paginate results by enabling **Pagination** in **Advance Properties** and specifying `page` and `page_size` as query parameters when calling the endpoint. For example, to get the second page with 10 items per page, you can use the following command: - - ```bash - curl --digest --user ':' \ - --request GET 'https://.data.tidbcloud.com/api/v1beta/app//endpoint/?page=2&page_size=10' - ``` - - Note that this feature is available only for `GET` requests where the last query is a `SELECT` statement. +- [TiDB Cloud Serverless branching (beta)](/tidb-cloud/branch-overview.md) introduces the following improvements to branch management: - For more information, see [Call an endpoint](/tidb-cloud/data-service-manage-endpoint.md#call-an-endpoint). + - **Flexible branch creation**: When creating a branch, you can select a specific cluster or branch as the parent and specify a precise point in time to use from the parent. This gives you precise control over the data in your branch. -- [Data Service (beta)](https://tidbcloud.com/console/data-service) supports caching endpoint response of `GET` requests for a specified time-to-live (TTL) period. + - **Branch reset**: You can reset a branch to synchronize it with the latest state of its parent. - This feature decreases database load and optimizes endpoint latency. + - **Improved GitHub integration**: The [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) GitHub App introduces the [`branch.mode`](/tidb-cloud/branch-github-integration.md#branchmode) parameter, which controls the behavior during pull request synchronization. In the default mode `reset`, the app resets the branch to match the latest changes in the pull request. - For an endpoint using the `GET` request method, you can enable **Cache Response** and configure the TTL period for the cache in **Advance Properties**. + For more information, see [Manage TiDB Cloud Serverless Branches](/tidb-cloud/branch-manage.md) and [Integrate TiDB Cloud Serverless Branching (Beta) with GitHub](/tidb-cloud/branch-github-integration.md). - For more information, see [Advanced properties](/tidb-cloud/data-service-manage-endpoint.md#advanced-properties). +## November 12, 2024 -- Disable the load balancing improvement for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters that are hosted on AWS and created after August 15, 2023, including: +**General changes** - - Disable automatically migrating existing connections to new TiDB nodes when you scale out TiDB nodes hosted on AWS. - - Disable automatically migrating existing connections to available TiDB nodes when you scale in TiDB nodes hosted on AWS. +- Add the pause duration limit for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters. - This change avoids resource contention of hybrid deployments and does not affect existing clusters with this improvement enabled. If you want to enable the load balancing improvement for your new clusters, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). + TiDB Cloud Dedicated now limits the maximum pause duration to 7 days. If you do not manually resume the cluster within 7 days, TiDB Cloud will automatically resume it. -## August 8, 2023 + This change applies only to **organizations created after November 12, 2024**. Organizations created on or before this date will gradually transition to the new pause behavior with prior notification. -**General changes** + For more information, see [Pause or Resume a TiDB Cloud Dedicated Cluster](/tidb-cloud/pause-or-resume-tidb-cluster.md). -- [Data Service (beta)](https://tidbcloud.com/console/data-service) now supports Basic Authentication. +- [Datadog integration (beta)](/tidb-cloud/monitor-datadog-integration.md) adds support for a new region: `AP1` (Japan). - You can provide your public key as the username and private key as the password in requests using the ['Basic' HTTP Authentication](https://datatracker.ietf.org/doc/html/rfc7617). Compared with Digest Authentication, the Basic Authentication is simpler, enabling more straightforward usage when calling Data Service endpoints. +- Support a new AWS region for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters: `Mumbai (ap-south-1)`. - For more information, see [Call an endpoint](/tidb-cloud/data-service-manage-endpoint.md#call-an-endpoint). +- Remove support for the AWS `São Paulo (sa-east-1)` region for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters. -## August 1, 2023 +## October 29, 2024 **General changes** -- Support the OpenAPI Specification for Data Apps in TiDB Cloud [Data Service](https://tidbcloud.com/console/data-service). - - TiDB Cloud Data Service provides autogenerated OpenAPI documentation for each Data App. In the documentation, you can view the endpoints, parameters, and responses, and try out the endpoints. - - You can also download an OpenAPI Specification (OAS) for a Data App and its deployed endpoints in YAML or JSON format. The OAS provides standardized API documentation, simplified integration, and easy code generation, which enables faster development and improved collaboration. +- New metric: add `tidbcloud_changefeed_checkpoint_ts` for Prometheus integration. - For more information, see [Use the OpenAPI Specification](/tidb-cloud/data-service-manage-data-app.md#use-the-openapi-specification) and [Use the OpenAPI Specification with Next.js](/tidb-cloud/data-service-oas-with-nextjs.md). + This metric tracks the checkpoint timestamp of a changefeed, representing the largest TSO (Timestamp Oracle) successfully written to the downstream. For more information on available metrics, see [Integrate TiDB Cloud with Prometheus and Grafana (Beta)](/tidb-cloud/monitor-prometheus-and-grafana-integration.md#metrics-available-to-prometheus). -- Support running Data App in [Postman](https://www.postman.com/). +## October 22, 2024 - The Postman integration empowers you to import a Data App's endpoints as a collection into your preferred workspace. Then you can benefit from enhanced collaboration and seamless API testing with support for both Postman web and desktop apps. - - For more information, see [Run Data App in Postman](/tidb-cloud/data-service-postman-integration.md). +**General changes** -- Introduce a new **Pausing** status for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters, allowing cost-effective pauses with no charges during this period. +- Upgrade the default TiDB version of new [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters from [v7.5.3](https://docs.pingcap.com/tidb/v7.5/release-7.5.3) to [v7.5.4](https://docs.pingcap.com/tidb/v7.5/release-7.5.4). - When you click **Pause** for a TiDB Dedicated cluster, the cluster will enter the **Pausing** status first. Once the pause operation is completed, the cluster status will transition to **Paused**. +## October 15, 2024 - A cluster can only be resumed after its status transitions to **Paused**, which resolves the abnormal resumption issue caused by rapid clicks of **Pause** and **Resume**. +**API changes** - For more information, see [Pause or resume a TiDB Dedicated cluster](/tidb-cloud/pause-or-resume-tidb-cluster.md). +* [MSP](https://docs.pingcap.com/tidbcloud/api/v1beta1/msp) is deprecated as of October 15, 2024, and will be removed in the future. If you are currently using the MSP API, migrate to the Partner Management API in [TiDB Cloud Partner](https://partner-console.tidbcloud.com/signin). -## July 26, 2023 +## September 24, 2024 **General changes** -- Introduce a powerful feature in TiDB Cloud [Data Service](https://tidbcloud.com/console/data-service): Automatic endpoint generation. +- Provide a new [TiFlash vCPU and RAM size](/tidb-cloud/size-your-cluster.md#tiflash-vcpu-and-ram) for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters hosted on AWS: `32 vCPU, 128 GiB` - Developers can now effortlessly create HTTP endpoints with minimal clicks and configurations. Eliminate repetitive boilerplate code, simplify and accelerate endpoint creation, and reduce potential errors. +**CLI changes** - For more information on how to use this feature, see [Generate an endpoint automatically](/tidb-cloud/data-service-manage-endpoint.md#generate-an-endpoint-automatically). +- Release [TiDB Cloud CLI v1.0.0-beta.2](https://github.com/tidbcloud/tidbcloud-cli/releases/tag/v1.0.0-beta.2). -- Support `PUT` and `DELETE` request methods for endpoints in TiDB Cloud [Data Service](https://tidbcloud.com/console/data-service). + TiDB Cloud CLI provides the following new features: - - Use the `PUT` method to update or modify data, similar to an `UPDATE` statement. - - Use the `DELETE` method to delete data, similar to a `DELETE` statement. + - Support SQL user management for [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters via [`ticloud serverless sql-user`](/tidb-cloud/ticloud-serverless-sql-user-create.md). + - Allow disabling the public endpoint for [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters in [`ticloud serverless create`](/tidb-cloud/ticloud-cluster-create.md) and [`ticloud serverless update`](/tidb-cloud/ticloud-serverless-update.md). + - Add the [`ticloud auth whoami`](/tidb-cloud/ticloud-auth-whoami.md) command to get information about the current user when using OAuth authentication. + - Support `--sql`, `--where`, and `--filter` flags in [`ticloud serverless export create`](/tidb-cloud/ticloud-serverless-export-create.md) to choose source tables flexibly. + - Support exporting data to CSV and Parquet files. + - Support exporting data to Amazon S3 using role ARN as credentials, and also support exporting to Google Cloud Storage and Azure Blob Storage. + - Support importing data from Amazon S3, Google Cloud Storage, and Azure Blob Storage. + - Support creating a branch from a branch and a specific timestamp. - For more information, see [Configure properties](/tidb-cloud/data-service-manage-endpoint.md#configure-properties). + TiDB Cloud CLI enhances the following features: -- Support **Batch Operation** for `POST`, `PUT`, and `DELETE` request methods in TiDB Cloud [Data Service](https://tidbcloud.com/console/data-service). + - Improve debug logging. Now it can log credentials and user-agent. + - Speed up local export file downloads from tens of KiB per second to tens of MiB per second. - When **Batch Operation** is enabled for an endpoint, you gain the ability to perform operations on multiple rows in a single request. For instance, you can insert multiple rows of data using a single `POST` request. + TiDB Cloud CLI replaces or removes the following features: - For more information, see [Advanced properties](/tidb-cloud/data-service-manage-endpoint.md#advanced-properties). + - The `--s3.bucket-uri` flag is replaced by `--s3.uri` in [`ticloud serverless export create`](/tidb-cloud/ticloud-serverless-export-create.md). + - The `--database` and `--table` flags are removed in [`ticloud serverless export create`](/tidb-cloud/ticloud-serverless-export-create.md). Instead, you can use `--sql`, `--where`, and `--filter` flags. + - [`ticloud serverless update`](/tidb-cloud/ticloud-serverless-update.md) cannot update the annotations field anymore. -## July 25, 2023 +## September 10, 2024 **General changes** -- Upgrade the default TiDB version of new [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters from [v6.5.3](https://docs.pingcap.com/tidb/v6.5/release-6.5.3) to [v7.1.1](https://docs.pingcap.com/tidb/v7.1/release-7.1.1). - -**Console changes** - -- Simplify access to PingCAP Support for TiDB Cloud users by optimizing support entries. Improvements include: +- Launch the TiDB Cloud Partner Web Console and Open API to enhance resource and billing management for TiDB Cloud partners. - - Add an entrance for **Support** in the in the lower-left corner. - - Revamp the menus of the **?** icon in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com/) to make them more intuitive. + Managed Service Providers (MSPs) and resellers through AWS Marketplace Channel Partner Private Offer (CPPO) can now leverage the [TiDB Cloud Partner Web Console](https://partner-console.tidbcloud.com/) and Open API to streamline their daily operations. - For more information, see [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). - -## July 18, 2023 - -**General changes** + For more information, see [TiDB Cloud Partner Web Console](/tidb-cloud/tidb-cloud-partners.md). -- Refine role-based access control at both the organization level and project level, which lets you grant roles with minimum permissions to users for better security, compliance, and productivity. +## September 3, 2024 - - The organization roles include `Organization Owner`, `Organization Billing Admin`, `Organization Console Audit Admin`, and `Organization Member`. - - The project roles include `Project Owner`, `Project Data Access Read-Write`, and `Project Data Access Read-Only`. - - To manage clusters in a project (such as cluster creation, modification, and deletion), you need to be in the `Organization Owner` or `Project Owner` role. - - For more information about permissions of different roles, see [User roles](/tidb-cloud/manage-user-access.md#user-roles). +**Console changes** -- Support the Customer-Managed Encryption Key (CMEK) feature (beta) for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters hosted on AWS. +- Support exporting data from TiDB Cloud Serverless clusters using the [TiDB Cloud console](https://tidbcloud.com/). - You can create CMEK based on AWS KMS to encrypt data stored in EBS and S3 directly from the TiDB Cloud console. This ensures that customer data is encrypted with a key managed by the customer, which enhances security. + Previously, TiDB Cloud only supported exporting data using the [TiDB Cloud CLI](/tidb-cloud/cli-reference.md). Now, you can easily export data from TiDB Cloud Serverless clusters to local files and Amazon S3 in the [TiDB Cloud console](https://tidbcloud.com/). - Note that this feature still has restrictions and is only available upon request. To apply for this feature, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). - -- Optimize the Import feature in TiDB Cloud, aimed at enhancing the data import experience. The following improvements have been made: + For more information, see [Export Data from TiDB Cloud Serverless](/tidb-cloud/serverless-export.md) and [Configure External Storage Access for TiDB Cloud Serverless](/tidb-cloud/serverless-external-storage.md). - - Unified Import entry for TiDB Serverless: consolidate the entries for importing data, allowing you to seamlessly switch between importing local files and importing files from Amazon S3. - - Streamlined configuration: importing data from Amazon S3 now only requires a single step, saving time and effort. - - Enhanced CSV configuration: the CSV configuration settings are now located under the file type option, making it easier for you to quickly configure the necessary parameters. - - Enhanced target table selection: support choosing the desired target tables for data import by clicking checkboxes. This improvement eliminates the need for complex expressions and simplifies the target table selection. - - Refined display information: resolve issues related to inaccurate information displayed during the import process. In addition, the Preview feature has been removed to prevent incomplete data display and avoid misleading information. - - Improved source files mapping: support defining mapping relationships between source files and target tables. It addresses the challenge of modifying source file names to meet specific naming requirements. - -## July 11, 2023 - -**General changes** +- Enhance the connection experience for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters. -- [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) now is Generally Available. - -- Introduce TiDB Bot (beta), an OpenAI-powered chatbot that offers multi-language support, 24/7 real-time response, and integrated documentation access. - - TiDB Bot provides you with the following benefits: - - - Continuous support: always available to assist and answer your questions for an enhanced support experience. - - Improved efficiency: automated responses reduce latency, improving overall operations. - - Seamless documentation access: direct access to TiDB Cloud documentation for easy information retrieval and quick issue resolution. - - To use TiDB Bot, click **?** in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com), and select **Ask TiDB Bot** to start a chat. - -- Support [the branching feature (beta)](/tidb-cloud/branch-overview.md) for [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. - - TiDB Cloud lets you create branches for TiDB Serverless clusters. A branch for a cluster is a separate instance that contains a diverged copy of data from the original cluster. It provides an isolated environment, allowing you to connect to it and experiment freely without worrying about affecting the original cluster. - - You can create branches for TiDB Serverless clusters created after July 5, 2023 by using either [TiDB Cloud console](/tidb-cloud/branch-manage.md) or [TiDB Cloud CLI](/tidb-cloud/ticloud-branch-create.md). - - If you use GitHub for application development, you can integrate TiDB Serverless branching into your GitHub CI/CD pipeline, which lets you automatically test your pull requests with branches without affecting the production database. For more information, see [Integrate TiDB Serverless Branching (Beta) with GitHub](/tidb-cloud/branch-github-integration.md). - -- Support weekly backup for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. For more information, see [Back up and restore TiDB Dedicated data](/tidb-cloud/backup-and-restore.md#automatic-backup). - -## July 4, 2023 + - Revise the **Connect** dialog interface to provide TiDB Cloud Dedicated users with a more streamlined and efficient connection experience. + - Introduce a new cluster-level **Networking** page to simplify network configuration for your cluster. + - Replace the **Security Settings** page with a new **Password Settings** page and move IP access list settings to the new **Networking** page. + + For more information, see [Connect to TiDB Cloud Dedicated](/tidb-cloud/connect-to-tidb-cluster.md). -**General changes** +- Enhance the data import experience for [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) and [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters: -- Support point-in-time recovery (PITR) (beta) for [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. + - Refine the layout of the **Import** page with a clearer layout. + - Unify the import steps for TiDB Cloud Serverless and TiDB Cloud Dedicated clusters. + - Simplify the AWS Role ARN creation process for easier connection setup. - You can now restore your TiDB Serverless cluster to any point in time within the last 90 days. This feature enhances the data recovery capability of TiDB Serverless clusters. For example, you can use PITR when data write errors occur and you want to restore the data to an earlier state. + For more information, see [Import data from files to TiDB Cloud](/tidb-cloud/tidb-cloud-migration-overview.md#import-data-from-files-to-tidb-cloud). - For more information, see [Back up and restore TiDB Serverless data](/tidb-cloud/backup-and-restore-serverless.md#restore). +## August 20, 2024 **Console changes** -- Enhance the **Usage This Month** panel on the cluster overview page for [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters to provide a clearer view of your current resource usage. - -- Enhance the overall navigation experience by making the following changes: +- Refine the layout of the **Create Private Endpoint Connection** page to improve the user experience for creating new private endpoint connections in [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters. - - Consolidate **Organization** and **Account** in the upper-right corner into the left navigation bar. - - Consolidate **Admin** in the left navigation bar into **Project** in the left navigation bar, and remove the ☰ hover menu in the upper-left corner. Now you can click to switch between projects and modify project settings. - - Consolidate all the help and support information for TiDB Cloud into the menu of the **?** icon in the lower-right corner, such as documentation, interactive tutorials, self-paced training, and support entries. + For more information, see [Connect to a TiDB Cloud Dedicated Cluster via Private Endpoint with AWS](/tidb-cloud/set-up-private-endpoint-connections.md) and [Connect to a TiDB Cloud Dedicated Cluster via Google Cloud Private Service Connect](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md). -- TiDB Cloud console now supports Dark Mode, which provides a more comfortable, eye-friendly experience. You can switch between light mode and dark mode from the bottom of the left navigation bar. - -## June 27, 2023 +## August 6, 2024 **General changes** -- Remove the pre-built sample dataset for newly created [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. - -## June 20, 2023 - -**General changes** - -- Upgrade the default TiDB version of new [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters from [v6.5.2](https://docs.pingcap.com/tidb/v6.5/release-6.5.2) to [v6.5.3](https://docs.pingcap.com/tidb/v6.5/release-6.5.3). - -## June 13, 2023 - -**General changes** - -- Support using changefeeds to stream data to Amazon S3. - - This enables seamless integration between TiDB Cloud and Amazon S3. It allows real-time data capture and replication from [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters to Amazon S3, ensuring that downstream applications and analytics have access to up-to-date data. - - For more information, see [Sink to cloud storage](/tidb-cloud/changefeed-sink-to-cloud-storage.md). - -- Increase the maximum node storage of 16 vCPU TiKV for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters from 4 TiB to 6 TiB. - - This enhancement increases the data storage capacity of your TiDB Dedicated cluster, improves workload scaling efficiency, and accommodates growing data requirements. +- [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) billing changes for load balancing on AWS. - For more information, see [Size your cluster](/tidb-cloud/size-your-cluster.md). + Starting from August 1, 2024, TiDB Cloud Dedicated bills include new AWS charges for public IPv4 addresses, aligned with [AWS pricing changes effective from February 1, 2024](https://aws.amazon.com/blogs/aws/new-aws-public-ipv4-address-charge-public-ip-insights/). The charge for each public IPv4 address is $0.005 per hour, which will result in approximately $10 per month for each TiDB Cloud Dedicated cluster hosted on AWS. -- Extend the [monitoring metrics retention period](/tidb-cloud/built-in-monitoring.md#metrics-retention-policy) for [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters from 3 days to 7 days. + This charge will appear under the existing **TiDB Cloud Dedicated - Data Transfer - Load Balancing** service in your [billing details](/tidb-cloud/tidb-cloud-billing.md#billing-details). - By extending the metrics retention period, now you have access to more historical data. This helps you identify trends and patterns of the cluster for better decision-making and faster troubleshooting. +- Upgrade the default TiDB version of new [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters from [v7.5.2](https://docs.pingcap.com/tidb/v7.5/release-7.5.2) to [v7.5.3](https://docs.pingcap.com/tidb/v7.5/release-7.5.3). **Console changes** -- Release a new native web infrastructure for the [**Key Visualizer**](/tidb-cloud/tune-performance.md#key-visualizer) page of [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. +- Enhance the cluster size configuration experience for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). - With the new infrastructure, you can easily navigate through the **Key Visualizer** page and access the necessary information in a more intuitive and efficient manner. The new infrastructure also resolves many problems on UX, making the SQL diagnosis process more user-friendly. + Refine the layout of the **Cluster Size** section on the [**Create Cluster**](/tidb-cloud/create-tidb-cluster.md) and [**Modify Cluster**](/tidb-cloud/scale-tidb-cluster.md) pages for TiDB Cloud Dedicated clusters. In addition, the **Cluster Size** section now includes links to node size recommendation documents, which helps you select an appropriate cluster size. -## June 6, 2023 +## July 23, 2024 **General changes** -- Introduce [Index Insight (beta)](/tidb-cloud/index-insight.md) for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters, which optimizes query performance by providing index recommendations for slow queries. +- [Data Service (beta)](https://tidbcloud.com/console/data-service) supports automatically generating vector search endpoints. - With Index Insight, you can improve the overall application performance and efficiency of your database operations in the following ways: + If your table contains [vector data types](/tidb-cloud/vector-search-data-types.md), you can automatically generate a vector search endpoint that calculates vector distances based on your selected distance function. - - Enhanced query performance: Index Insight identifies slow queries and suggests appropriate indexes for them, thereby speeding up query execution, reducing response time, and improving user experience. - - Cost efficiency: By using Index Insight to optimize query performance, the need for extra computing resources is reduced, enabling you to use existing infrastructure more effectively. This can potentially lead to operational cost savings. - - Simplified optimization process: Index Insight simplifies the identification and implementation of index improvements, eliminating the need for manual analysis and guesswork. As a result, you can save time and effort with accurate index recommendations. - - Improved application efficiency: By using Index Insight to optimize database performance, applications running on TiDB Cloud can handle larger workloads and serve more users concurrently, which makes scaling operations of applications more efficient. + This feature enables seamless integration with AI platforms such as [Dify](https://docs.dify.ai/guides/tools) and [GPTs](https://openai.com/blog/introducing-gpts), enhancing your applications with advanced natural language processing and AI capabilities for more complex tasks and intelligent solutions. - To use Index Insight, navigate to the **Diagnosis** page of your TiDB Dedicated cluster and click the **Index Insight BETA** tab. + For more information, see [Generate an endpoint automatically](/tidb-cloud/data-service-manage-endpoint.md#generate-an-endpoint-automatically) and [Integrate a Data App with Third-Party Tools](/tidb-cloud/data-service-integrations.md). - For more information, see [Use Index Insight (beta)](/tidb-cloud/index-insight.md). +- Introduce the budget feature to help you track actual TiDB Cloud costs against planned expenses, preventing unexpected costs. -- Introduce [TiDB Playground](https://play.tidbcloud.com/?utm_source=docs&utm_medium=tidb_cloud_release_notes), an interactive platform for experiencing the full capabilities of TiDB, without registration or installation. + To access this feature, you must be in the `Organization Owner` or `Organization Billing Admin` role of your organization. - TiDB Playground is an interactive platform designed to provide a one-stop-shop experience for exploring the capabilities of TiDB, such as scalability, MySQL compatibility, and real-time analytics. + For more information, see [Manage budgets for TiDB Cloud](/tidb-cloud/tidb-cloud-budget.md). - With TiDB Playground, you can try out TiDB features in a controlled environment free from complex configurations in real-time, making it ideal to understand the features in TiDB. - - To get started with TiDB Playground, go to the [**TiDB Playground**](https://play.tidbcloud.com/?utm_source=docs&utm_medium=tidb_cloud_release_notes) page, select a feature you want to explore, and begin your exploration. - -## June 5, 2023 - -**General changes** - -- Support connecting your [Data App](/tidb-cloud/tidb-cloud-glossary.md#data-app) to GitHub. - - By [connecting your Data App to GitHub](/tidb-cloud/data-service-manage-github-connection.md), you can manage all configurations of the Data App as [code files](/tidb-cloud/data-service-app-config-files.md) on Github, which integrates TiDB Cloud Data Service seamlessly with your system architecture and DevOps process. - - With this feature, you can easily accomplish the following tasks, which improves the CI/CD experience of developing Data Apps: - - - Automatically deploy Data App changes with GitHub. - - Configure CI/CD pipelines of your Data App changes on GitHub with version control. - - Disconnect from a connected GitHub repository. - - Review endpoint changes before the deployment. - - View deployment history and take necessary actions in the event of a failure. - - Re-deploy a commit to roll back to an earlier deployment. - - For more information, see [Deploy Data App automatically with GitHub](/tidb-cloud/data-service-manage-github-connection.md). - -## June 2, 2023 +## July 9, 2024 **General changes** -- In our pursuit to simplify and clarify, we have updated the names of our products: - - - "TiDB Cloud Serverless Tier" is now called "TiDB Serverless". - - "TiDB Cloud Dedicated Tier" is now called "TiDB Dedicated". - - "TiDB On-Premises" is now called "TiDB Self-Hosted". +- Enhance the [System Status](https://status.tidbcloud.com/) page to provide better insights into TiDB Cloud system health and performance. - Enjoy the same great performance under these refreshed names. Your experience is our priority. - -## May 30, 2023 - -**General changes** - -- Enhance support for incremental data migration for the Data Migration feature in TiDB Cloud. - - You can now specify a binlog position or a global transaction identifier (GTID) to replicate only incremental data generated after the specified position to TiDB Cloud. This enhancement empowers you with greater flexibility to select and replicate the data you need, aligning with your specific requirements. - - For details, refer to [Migrate Only Incremental Data from MySQL-Compatible Databases to TiDB Cloud Using Data Migration](/tidb-cloud/migrate-incremental-data-from-mysql-using-data-migration.md). - -- Add a new event type (`ImportData`) to the [**Events**](/tidb-cloud/tidb-cloud-events.md) page. - -- Remove **Playground** from the TiDB Cloud console. - - Stay tuned for the new standalone Playground with an optimized experience. - -## May 23, 2023 - -**General changes** - -- When uploading a CSV file to TiDB, you can use not only English letters and numbers, but also characters such as Chinese and Japanese to define column names. However, for special characters, only underscore (`_`) is supported. - - For details, refer to [Import Local Files to TiDB Cloud](/tidb-cloud/tidb-cloud-import-local-files.md). - -## May 16, 2023 + To access it, visit directly, or navigate via the [TiDB Cloud console](https://tidbcloud.com) by clicking **?** in the lower-right corner and selecting **System Status**. **Console changes** -- Introduce the left navigation entries organized by functional categories for both Dedicated and Serverless tiers. - - The new navigation makes it easier and more intuitive for you to discover the feature entries. To view the new navigation, access the overview page of your cluster. +- Refine the **VPC Peering** page layout to improve the user experience for [creating VPC Peering connections](/tidb-cloud/set-up-vpc-peering-connections.md) in [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters. -- Release a new native web infrastructure for the following two tabs on the **Diagnosis** page of Dedicated Tier clusters. - - - [Slow Query](/tidb-cloud/tune-performance.md#slow-query) - - [SQL Statement](/tidb-cloud/tune-performance.md#statement-analysis) - - With the new infrastructure, you can easily navigate through the two tabs and access the necessary information in a more intuitive and efficient manner. The new infrastructure also improves user experience, making the SQL diagnosis process more user-friendly. - -## May 9, 2023 +## July 2, 2024 **General changes** -- Support changing node sizes for GCP-hosted clusters created after April 26, 2023. - - With this feature, you can upgrade to higher-performance nodes for increased demand or downgrade to lower-performance nodes for cost savings. With this added flexibility, you can adjust your cluster's capacity to align with your workloads and optimize costs. - - For detailed steps, see [Change node size](/tidb-cloud/scale-tidb-cluster.md#change-vcpu-and-ram). - -- Support importing compressed files. You can import CSV and SQL files in the following formats: `.gzip`, `.gz`, `.zstd`, `.zst`, and `.snappy`. This feature provides a more efficient and cost-effective way to import data and reduces your data transfer costs. - - For more information, see [Import CSV Files from Amazon S3 or GCS into TiDB Cloud](/tidb-cloud/import-csv-files.md) and [Import Sample Data](/tidb-cloud/import-sample-data.md). - -- Support AWS PrivateLink-powered endpoint connection as a new network access management option for TiDB Cloud [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. - - The private endpoint connection does not expose your data to the public internet. In addition, the endpoint connection supports CIDR overlap and is easier for network management. - - For more information, see [Set Up Private Endpoint Connections](/tidb-cloud/set-up-private-endpoint-connections.md). - -**Console changes** - -- Add new event types to the [**Event**](/tidb-cloud/tidb-cloud-events.md) page to record backup, restore, and changefeed actions for [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. +- [Data Service (beta)](https://tidbcloud.com/console/data-service) provides an endpoint library with predefined system endpoints that you can directly add to your Data App, reducing the effort in your endpoint development. - To get a full list of the events that can be recorded, see [Logged events](/tidb-cloud/tidb-cloud-events.md#logged-events). + Currently, the library only includes the `/system/query` endpoint, which enables you to execute any SQL statement by simply passing the statement in the predefined `sql` parameter. This endpoint facilitates the immediate execution of SQL queries, enhancing flexibility and efficiency. -- Introduce the **SQL Statement** tab on the [**SQL Diagnosis**](/tidb-cloud/tune-performance.md) page for [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. + For more information, see [Add a predefined system endpoint](/tidb-cloud/data-service-manage-endpoint.md#add-a-predefined-system-endpoint). - The **SQL Statement** tab provides the following: +- Enhance slow query data storage. - - A comprehensive overview of all SQL statements executed by your TiDB database, allowing you to easily identify and diagnose slow queries. - - Detailed information on each SQL statement, such as the query time, execution plan, and the database server response, helping you optimize your database performance. - - A user-friendly interface that makes it easy to sort, filter, and search through large amounts of data, enabling you to focus on the most critical queries. + The slow query access on the [TiDB Cloud console](https://tidbcloud.com) is now more stable and does not affect database performance. - For more information, see [Statement Analysis](/tidb-cloud/tune-performance.md#statement-analysis). - -## May 6, 2023 +## June 25, 2024 **General changes** -- Support directly accessing the [Data Service endpoint](/tidb-cloud/tidb-cloud-glossary.md#endpoint) in the region where a TiDB [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster is located. - - For newly created Serverless Tier clusters, the endpoint URL now includes the cluster region information. By requesting the regional domain `.data.tidbcloud.com`, you can directly access the endpoint in the region where the TiDB cluster is located. - - Alternatively, you can also request the global domain `data.tidbcloud.com` without specifying a region. In this way, TiDB Cloud will internally redirect the request to the target region, but this might result in additional latency. If you choose this way, make sure to add the `--location-trusted` option to your curl command when calling an endpoint. - - For more information, see [Call an endpoint](/tidb-cloud/data-service-manage-endpoint.md#call-an-endpoint). - -## April 25, 2023 - -**General changes** - -- For the first five [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters in your organization, TiDB Cloud provides a free usage quota for each of them as follows: - - - Row storage: 5 GiB - - [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit): 50 million RUs per month - - Until May 31, 2023, Serverless Tier clusters are still free, with a 100% discount off. After that, usage beyond the free quota will be charged. - - You can easily [monitor your cluster usage or increase your usage quota](/tidb-cloud/manage-serverless-spend-limit.md#manage-spending-limit-for-tidb-serverless-clusters) in the **Usage This Month** area of your cluster **Overview** page. Once the free quota of a cluster is reached, the read and write operations on this cluster will be throttled until you increase the quota or the usage is reset upon the start of a new month. - - For more information about the RU consumption of different resources (including read, write, SQL CPU, and network egress), the pricing details, and the throttled information, see [TiDB Cloud Serverless Tier Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details). - -- Support backup and restore for TiDB Cloud [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. +- [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) supports vector search (beta). - For more information, see [Back up and Restore TiDB Cluster Data](/tidb-cloud/backup-and-restore-serverless.md). + The vector search (beta) feature provides an advanced search solution for performing semantic similarity searches across various data types, including documents, images, audio, and video. This feature enables developers to easily build scalable applications with generative artificial intelligence (AI) capabilities using familiar MySQL skills. Key features include: -- Upgrade the default TiDB version of new [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters from [v6.5.1](https://docs.pingcap.com/tidb/v6.5/release-6.5.1) to [v6.5.2](https://docs.pingcap.com/tidb/v6.5/release-6.5.2). + - [Vector data types](/tidb-cloud/vector-search-data-types.md), [vector index](/tidb-cloud/vector-search-index.md), and [vector functions and operators](/tidb-cloud/vector-search-functions-and-operators.md). + - Ecosystem integrations with [LangChain](/tidb-cloud/vector-search-integrate-with-langchain.md), [LlamaIndex](/tidb-cloud/vector-search-integrate-with-llamaindex.md), and [JinaAI](/tidb-cloud/vector-search-integrate-with-jinaai-embedding.md). + - Programming language support for Python: [SQLAlchemy](/tidb-cloud/vector-search-integrate-with-sqlalchemy.md), [Peewee](/tidb-cloud/vector-search-integrate-with-peewee.md), and [Django ORM](/tidb-cloud/vector-search-integrate-with-django-orm.md). + - Sample applications and tutorials: perform semantic searches for documents using [Python](/tidb-cloud/vector-search-get-started-using-python.md) or [SQL](/tidb-cloud/vector-search-get-started-using-sql.md). -- Provide a maintenance window feature to enable you to easily schedule and manage planned maintenance activities for [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. + For more information, see [Vector search (beta) overview](/tidb-cloud/vector-search-overview.md). - A maintenance window is a designated timeframe during which planned maintenance activities, such as operating system updates, security patches, and infrastructure upgrades, are performed automatically to ensure the reliability, security, and performance of the TiDB Cloud service. +- [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) now offers weekly email reports for organization owners. - During a maintenance window, temporary connection disruptions or QPS fluctuations might occur, but the clusters remain available, and SQL operations, the existing data import, backup, restore, migration, and replication tasks can still run normally. See [a list of allowed and disallowed operations](/tidb-cloud/configure-maintenance-window.md#allowed-and-disallowed-operations-during-a-maintenance-window) during maintenance. + These reports provide insights into the performance and activity of your clusters. By receiving automatic weekly updates, you can stay informed about your clusters and make data-driven decisions to optimize your clusters. - We will strive to minimize the frequency of maintenance. If a maintenance window is planned, the default start time is 03:00 Wednesday (based on the time zone of your TiDB Cloud organization) of the target week. To avoid potential disruptions, it is important to be aware of the maintenance schedules and plan your operations accordingly. +- Release Chat2Query API v3 endpoints and deprecate the Chat2Query API v1 endpoint `/v1/chat2data`. - - To keep you informed, TiDB Cloud will send you three email notifications for every maintenance window: one before, one starting, and one after the maintenance tasks. - - To minimize the maintenance impact, you can modify the maintenance start time to your preferred time or defer maintenance activities on the **Maintenance** page. + With Chat2Query API v3 endpoints, you can start multi-round Chat2Query by using sessions. - For more information, see [Configure maintenance window](/tidb-cloud/configure-maintenance-window.md). - -- Improve load balancing of TiDB and reduce connection drops when you scale TiDB nodes of [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters that are hosted on AWS and created after April 25, 2023. - - - Support automatically migrating existing connections to new TiDB nodes when you scale out TiDB nodes. - - Support automatically migrating existing connections to available TiDB nodes when you scale in TiDB nodes. - - Currently, this feature is provided for all Dedicated Tier clusters hosted on AWS. - -**Console changes** - -- Release a new native web infrastructure for the [Monitoring](/tidb-cloud/built-in-monitoring.md#view-the-metrics-page) page of [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. - - With the new infrastructure, you can easily navigate through the [Monitoring](/tidb-cloud/built-in-monitoring.md#view-the-metrics-page) page and access the necessary information in a more intuitive and efficient manner. The new infrastructure also resolves many problems on UX, making the monitoring process more user-friendly. - -## April 18, 2023 - -**General changes** - -- Support scaling up or down [Data Migration job specifications](/tidb-cloud/tidb-cloud-billing-dm.md#specifications-for-data-migration) for [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. - - With this feature, you can improve migration performance by scaling up specifications or reduce costs by scaling down specifications. - - For more information, see [Migrate MySQL-Compatible Databases to TiDB Cloud Using Data Migration](/tidb-cloud/migrate-from-mysql-using-data-migration.md#scale-a-migration-job-specification). + For more information, see [Get started with Chat2Query API](/tidb-cloud/use-chat2query-api.md). **Console changes** -- Revamp the UI to make [cluster creation](https://tidbcloud.com/console/clusters/create-cluster) experience more user-friendly, enabling you to create and configure clusters with just a few clicks. - - The new design focuses on simplicity, reducing visual clutter, and providing clear instructions. After clicking **Create** on the cluster creation page, you will be directed to the cluster overview page without having to wait for the cluster creation to be completed. +- Rename Chat2Query (beta) to SQL Editor (beta). - For more information, see [Create a cluster](/tidb-cloud/create-tidb-cluster.md). + The interface previously known as Chat2Query is renamed to SQL Editor. This change clarifies the distinction between manual SQL editing and AI-assisted query generation, enhancing usability and your overall experience. -- Introduce the **Discounts** tab on the **Billing** page to show the discount information for organization owners and billing administrators. + - **SQL Editor**: the default interface for manually writing and executing SQL queries in the TiDB Cloud console. + - **Chat2Query**: the AI-assisted text-to-query feature, which enables you to interact with your databases using natural language to generate, rewrite, and optimize SQL queries. - For more information, see [Discounts](/tidb-cloud/tidb-cloud-billing.md#discounts). + For more information, see [Explore your data with AI-assisted SQL Editor](/tidb-cloud/explore-data-with-chat2query.md). -## April 11, 2023 +## June 18, 2024 **General changes** -- Improve the load balance of TiDB and reduce connection drops when you scale TiDB nodes of [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters hosted on AWS. +- Increase the maximum node storage of 16 vCPU TiFlash and 32 vCPU TiFlash for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters from 2048 GiB to 4096 GiB. - - Support automatically migrating existing connections to new TiDB nodes when you scale out TiDB nodes. - - Support automatically migrating existing connections to available TiDB nodes when you scale in TiDB nodes. + This enhancement increases the analytics data storage capacity of your TiDB Cloud Dedicated cluster, improves workload scaling efficiency, and accommodates growing data requirements. - Currently, this feature is only provided for Dedicated Tier clusters that are hosted on the AWS `Oregon (us-west-2)` region. + For more information, see [TiFlash node storage](/tidb-cloud/size-your-cluster.md#tiflash-node-storage). -- Support the [New Relic](https://newrelic.com/) integration for [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. +- Upgrade the default TiDB version of new [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters from [v7.5.1](https://docs.pingcap.com/tidb/v7.5/release-7.5.1) to [v7.5.2](https://docs.pingcap.com/tidb/v7.5/release-7.5.2). - With the New Relic integration, you can configure TiDB Cloud to send metric data of your TiDB clusters to [New Relic](https://newrelic.com/). Then, you can monitor and analyze both your application performance and your TiDB database performance on [New Relic](https://newrelic.com/). This feature can help you quickly identify and troubleshoot potential issues and reduce the resolution time. +## June 4, 2024 - For integration steps and available metrics, see [Integrate TiDB Cloud with New Relic](/tidb-cloud/monitor-new-relic-integration.md). - -- Add the following [changefeed](/tidb-cloud/changefeed-overview.md) metrics to the Prometheus integration for Dedicated Tier clusters. +**General changes** - - `tidbcloud_changefeed_latency` - - `tidbcloud_changefeed_replica_rows` +- Introduce the Recovery Group feature (beta) for disaster recovery of [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters deployed on AWS. - If you have [integrated TiDB Cloud with Prometheus](/tidb-cloud/monitor-prometheus-and-grafana-integration.md), you can monitor the performance and health of changefeeds in real time using these metrics. Additionally, you can easily create alerts to monitor the metrics using Prometheus. + This feature enables you to replicate your databases between TiDB Cloud Dedicated clusters, ensuring rapid recovery in the event of a regional disaster. If you are in the `Project Owner` role, you can enable this feature by creating a new recovery group and assigning databases to the group. By replicating databases with recovery groups, you can improve disaster readiness, meet stricter availability SLAs, and achieve more aggressive Recovery Point Objectives (RPO) and Recovery Time Objectives (RTO). -**Console changes** + For more information, see [Get started with recovery groups](/tidb-cloud/recovery-group-get-started.md). -- Update the [Monitoring](/tidb-cloud/built-in-monitoring.md#view-the-metrics-page) page for [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters to use [node-level resource metrics](/tidb-cloud/built-in-monitoring.md#server). +- Introduce billing and metering (beta) for the [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) columnar storage [TiFlash](/tiflash/tiflash-overview.md). - With node-level resource metrics, you can see a more accurate representation of resource consumption to better understand the actual usage of purchased services. + Until June 30, 2024, columnar storage in TiDB Cloud Serverless clusters remains free with a 100% discount. After this date, each TiDB Cloud Serverless cluster will include a free quota of 5 GiB for columnar storage. Usage beyond the free quota will be charged. - To access these metrics, navigate to the [Monitoring](/tidb-cloud/built-in-monitoring.md#view-the-metrics-page) page of your cluster, and then check the **Server** category under the **Metrics** tab. + For more information, see [TiDB Cloud Serverless pricing details](https://www.pingcap.com/tidb-serverless-pricing-details/#storage). -- Optimize the [Billing](/tidb-cloud/tidb-cloud-billing.md#billing-details) page by reorganizing the billing items in **Summary by Project** and **Summary by Service**, which makes the billing information clearer. +- [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) supports [Time to live (TTL)](/time-to-live.md). -## April 4, 2023 +## May 28, 2024 **General changes** -- Remove the following two alerts from [TiDB Cloud built-in alerts](/tidb-cloud/monitor-built-in-alerting.md#tidb-cloud-built-in-alert-conditions) to prevent false positives. This is because temporary offline or out-of-memory (OOM) issues on one of the nodes do not significantly affect the overall health of a cluster. - - - At least one TiDB node in the cluster has run out of memory. - - One or more cluster nodes are offline. - -**Console changes** - -- Introduce the [Alerts](/tidb-cloud/monitor-built-in-alerting.md) page for [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters, which lists both active and closed alerts for each Dedicated Tier cluster. +- Google Cloud `Taiwan (asia-east1)` region supports the [Data Migration](/tidb-cloud/migrate-from-mysql-using-data-migration.md) feature. - The **Alerts** page provides the following: + The [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters hosted in the Google Cloud `Taiwan (asia-east1)` region now support the Data Migration (DM) feature. If your upstream data is stored in or near this region, you can now take advantage of faster and more reliable data migration from Google Cloud to TiDB Cloud. - - An intuitive and user-friendly user interface. You can view alerts for your clusters on this page even if you have not subscribed to the alert notification emails. - - Advanced filtering options to help you quickly find and sort alerts based on their severity, status, and other attributes. It also allows you to view the historical data for the last 7 days, which eases the alert history tracking. - - The **Edit Rule** feature. You can customize alert rule settings to meet your cluster's specific needs. +- Provide a new [TiDB node size](/tidb-cloud/size-your-cluster.md#tidb-vcpu-and-ram) for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters hosted on AWS and Google Cloud: `16 vCPU, 64 GiB` - For more information, see [TiDB Cloud built-in alerts](/tidb-cloud/monitor-built-in-alerting.md). - -- Consolidate the help-related information and actions of TiDB Cloud into a single place. +**API changes** - Now, you can get all the [TiDB Cloud help information](/tidb-cloud/tidb-cloud-support.md) and contact support by clicking **?** in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com/). +- Introduce TiDB Cloud Data Service API for managing the following resources automatically and efficiently: -- Introduce the [Getting Started](https://tidbcloud.com/console/getting-started) page to help you learn about TiDB Cloud. + * **Data App**: a collection of endpoints that you can use to access data for a specific application. + * **Data Source**: clusters linked to Data Apps for data manipulation and retrieval. + * **Endpoint**: a web API that you can customize to execute SQL statements. + * **Data API Key**: used for secure endpoint access. + * **OpenAPI Specification**: Data Service supports generating the OpenAPI Specification 3.0 for each Data App, which enables you to interact with your endpoints in a standardized format. - The **Getting Started** page provides you with interactive tutorials, essential guides, and useful links. By following interactive tutorials, you can easily explore TiDB Cloud features and HTAP capabilities with pre-built industry-specific datasets (Steam Game Dataset and S&P 500 Dataset). + These TiDB Cloud Data Service API endpoints are released in TiDB Cloud API v1beta1, which is the latest API version of TiDB Cloud. - To access the **Getting Started** page, click **Getting Started** in the left navigation bar of the [TiDB Cloud console](https://tidbcloud.com/). On this page, you can click **Query Sample Dataset** to open the interactive tutorials or click other links to explore TiDB Cloud. Alternatively, you can click **?** in the lower-right corner and click **Interactive Tutorials**. + For more information, see [API documentation (v1beta1)](https://docs.pingcap.com/tidbcloud/api/v1beta1/dataservice). -## March 29, 2023 +## May 21, 2024 **General changes** -- [Data Service (beta)](/tidb-cloud/data-service-overview.md) supports more fine-grained access control for Data Apps. - - On the Data App details page, now you can link clusters to your Data App and specify the role for each API key. The role controls whether the API key can read or write data to the linked clusters and can be set to `ReadOnly` or `ReadAndWrite`. This feature provides cluster-level and permission-level access control for Data Apps, giving you more flexibility to control the access scope according to your business needs. +- Provide a new [TiDB node size](/tidb-cloud/size-your-cluster.md#tidb-vcpu-and-ram) for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters hosted on Google Cloud: `8 vCPU, 16 GiB` - For more information, see [Manage linked clusters](/tidb-cloud/data-service-manage-data-app.md#manage-linked-data-sources) and [Manage API keys](/tidb-cloud/data-service-api-key.md). - -## March 28, 2023 +## May 14, 2024 **General changes** -- Add 2 RCUs, 4 RCUs, and 8 RCUs specifications for [changefeeds](/tidb-cloud/changefeed-overview.md), and support choosing your desired specification when you [create a changefeed](/tidb-cloud/changefeed-overview.md#create-a-changefeed). - - Using these new specifications, the data replication costs can be reduced by up to 87.5% compared to scenarios where 16 RCUs were previously required. - -- Support scaling up or down specifications for [changefeeds](/tidb-cloud/changefeed-overview.md) created after March 28, 2023. +- Expand the selection of time zones in the [**Time Zone**](/tidb-cloud/manage-user-access.md#set-the-time-zone-for-your-organization) section to better accommodate customers from diverse regions. - You can improve replication performance by choosing a higher specification or reduce replication costs by choosing a lower specification. +- Support [creating a VPC peering](/tidb-cloud/set-up-vpc-peering-connections.md) when your VPC is in a different region from the VPC of TiDB Cloud. - For more information, see [Scale a changefeed](/tidb-cloud/changefeed-overview.md#scale-a-changefeed). +- [Data Service (beta)](https://tidbcloud.com/console/data-service) supports path parameters alongside query parameters. -- Support replicating incremental data in real-time from a [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) cluster in AWS to a [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster in the same project and same region. + This feature enhances resource identification with structured URLs and improves user experience, search engine optimization (SEO), and client integration, offering developers more flexibility and better alignment with industry standards. - For more information, see [Sink to TiDB Cloud](/tidb-cloud/changefeed-sink-to-tidb-cloud.md). + For more information, see [Basic properties](/tidb-cloud/data-service-manage-endpoint.md#basic-properties). -- Support two new GCP regions for the [Data Migration](/tidb-cloud/migrate-from-mysql-using-data-migration.md) feature of [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters: `Singapore (asia-southeast1)` and `Oregon (us-west1)`. +## April 16, 2024 - With these new regions, you have more options for migrating your data to TiDB Cloud. If your upstream data is stored in or near these regions, you can now take advantage of faster and more reliable data migration from GCP to TiDB Cloud. - - For more information, see [Migrate MySQL-compatible databases to TiDB Cloud using Data Migration](/tidb-cloud/migrate-from-mysql-using-data-migration.md). +**CLI changes** -**Console changes** +- Introduce [TiDB Cloud CLI 1.0.0-beta.1](https://github.com/tidbcloud/tidbcloud-cli), built upon the new [TiDB Cloud API](/tidb-cloud/api-overview.md). The new CLI brings the following new features: -- Release a new native web infrastructure for the [Slow Query](/tidb-cloud/tune-performance.md#slow-query) page of [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. + - [Export data from TiDB Cloud Serverless clusters](/tidb-cloud/serverless-export.md) + - [Import data from local storage into TiDB Cloud Serverless clusters](/tidb-cloud/ticloud-import-start.md) + - [Authenticate via OAuth](/tidb-cloud/ticloud-auth-login.md) + - [Ask questions via TiDB Bot](/tidb-cloud/ticloud-ai.md) - With this new infrastructure, you can easily navigate through the [Slow Query](/tidb-cloud/tune-performance.md#slow-query) page and access the necessary information in a more intuitive and efficient manner. The new infrastructure also resolves many problems on UX, making the SQL diagnosis process more user-friendly. + Before upgrading your TiDB Cloud CLI, note that this new CLI is incompatible with previous versions. For example, `ticloud cluster` in CLI commands is now updated to `ticloud serverless`. For more information, see [TiDB Cloud CLI reference](/tidb-cloud/cli-reference.md). -## March 21, 2023 +## April 9, 2024 **General changes** -- Introduce [Data Service (beta)](https://tidbcloud.com/console/data-service) for [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters, which enables you to access data via an HTTPS request using a custom API endpoint. - - With Data Service, you can seamlessly integrate TiDB Cloud with any application or service that is compatible with HTTPS. The following are some common scenarios: - - - Access the database of your TiDB cluster directly from a mobile or web application. - - Use serverless edge functions to call endpoints and avoid scalability issues caused by database connection pooling. - - Integrate TiDB Cloud with data visualization projects by using Data Service as a data source. - - Connect to your database from an environment that MySQL interface does not support. - - In addition, TiDB Cloud provides the [Chat2Query API](/tidb-cloud/use-chat2query-api.md), a RESTful interface that allows you to generate and execute SQL statements using AI. - - To access Data Service, navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page in the left navigation pane. For more information, see the following documentation: - - - [Data Service Overview](/tidb-cloud/data-service-overview.md) - - [Get Started with Data Service](/tidb-cloud/data-service-get-started.md) - - [Get Started with Chat2Query API](/tidb-cloud/use-chat2query-api.md) - -- Support decreasing the size of TiDB, TiKV, and TiFlash nodes to scale in a [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) cluster that is hosted on AWS and created after December 31, 2022. - - You can decrease the node size [via the TiDB Cloud console](/tidb-cloud/scale-tidb-cluster.md#change-vcpu-and-ram) or [via the TiDB Cloud API (beta)](https://docs.pingcap.com/tidbcloud/api/v1beta#tag/Cluster/operation/UpdateCluster). - -- Support a new GCP region for the [Data Migration](/tidb-cloud/migrate-from-mysql-using-data-migration.md) feature of [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters: `Tokyo (asia-northeast1)`. - - The feature can help you migrate data from MySQL-compatible databases in Google Cloud Platform (GCP) to your TiDB cluster easily and efficiently. - - For more information, see [Migrate MySQL-compatible databases to TiDB Cloud using Data Migration](/tidb-cloud/migrate-from-mysql-using-data-migration.md). +- Provide a new [TiDB node size](/tidb-cloud/size-your-cluster.md#tidb-vcpu-and-ram) for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters hosted on AWS: `8 vCPU, 32 GiB`. -**Console changes** - -- Introduce the **Events** page for [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters, which provides the records of main changes to your cluster. - - On this page, you can view the event history for the last 7 days and track important details such as the trigger time and the user who initiated an action. For example, you can view events such as when a cluster was paused or who modified the cluster size. - - For more information, see [TiDB Cloud cluster events](/tidb-cloud/tidb-cloud-events.md). - -- Add the **Database Status** tab to the **Monitoring** page for [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters, which displays the following database-level metrics: - - - QPS Per DB - - Average Query Duration Per DB - - Failed Queries Per DB - - With these metrics, you can monitor the performance of individual databases, make data-driven decisions, and take actions to improve the performance of your applications. - - For more information, see [Monitoring metrics for Serverless Tier clusters](/tidb-cloud/built-in-monitoring.md). - -## March 14, 2023 +## April 2, 2024 **General changes** -- Upgrade the default TiDB version of new [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters from [v6.5.0](https://docs.pingcap.com/tidb/v6.5/release-6.5.0) to [v6.5.1](https://docs.pingcap.com/tidb/v6.5/release-6.5.1). - -- Support modifying column names of the target table to be created by TiDB Cloud when uploading a local CSV file with a header row. +- Introduce two service plans for [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters: **Free** and **Scalable**. - When importing a local CSV file with a header row to a [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster, if you need TiDB Cloud to create the target table and the column names in the header row do not follow the TiDB Cloud column naming conventions, you will see a warning icon next to the corresponding column name. To resolve the warning, you can move the cursor over the icon and follow the message to edit the existing column names or enter new column names. + To meet different user requirements, TiDB Cloud Serverless offers the free and scalable service plans. Whether you are just getting started or scaling to meet the increasing application demands, these plans provide the flexibility and capabilities you need. - For information about column naming conventions, see [Import local files](/tidb-cloud/tidb-cloud-import-local-files.md#import-local-files). + For more information, see [Cluster plans](/tidb-cloud/select-cluster-tier.md#cluster-plans). -## March 7, 2023 +- Modify the throttling behavior for TiDB Cloud Serverless clusters upon reaching their usage quota. Now, once a cluster reaches its usage quota, it immediately denies any new connection attempts, thereby ensuring uninterrupted service for existing operations. -**General changes** - -- Upgrade the default TiDB version of all [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters from [v6.4.0](https://docs.pingcap.com/tidb/v6.4/release-6.4.0) to [v6.6.0](https://docs.pingcap.com/tidb/v6.6/release-6.6.0). + For more information, see [Usage quota](/tidb-cloud/serverless-limitations.md#usage-quota). -## February 28, 2023 +## March 5, 2024 **General changes** -- Add the [SQL Diagnosis](/tidb-cloud/tune-performance.md) feature for [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. - - With SQL Diagnosis, you can gain deep insights into SQL-related runtime status, which makes the SQL performance tuning more efficient. Currently, the SQL Diagnosis feature for Serverless Tier only provides slow query data. - - To use SQL Diagnosis, click **SQL Diagnosis** on the left navigation bar of your Serverless Tier cluster page. +- Upgrade the default TiDB version of new [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters from [v7.5.0](https://docs.pingcap.com/tidb/v7.5/release-7.5.0) to [v7.5.1](https://docs.pingcap.com/tidb/v7.5/release-7.5.1). **Console changes** -- Optimize the left navigation. +- Introduce the **Cost Explorer** tab on the [**Billing**](https://tidbcloud.com/console/org-settings/billing/payments) page, which provides an intuitive interface for analyzing and customizing cost reports for your organization over time. - You can navigate pages more efficiently, for example: + To use this feature, navigate to the **Billing** page of your organization and click the **Cost Explorer** tab. - - You can hover the mouse in the upper-left corner to quickly switch between clusters or projects. - - You can switch between the **Clusters** page and the **Admin** page. + For more information, see [Cost Explorer](/tidb-cloud/tidb-cloud-billing.md#cost-explorer). -**API changes** +- [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) displays a **limit** label for [node-level resource metrics](/tidb-cloud/built-in-monitoring.md#server). -- Release several TiDB Cloud API endpoints for data import: + The **limit** label shows the maximum usage of resources such as CPU, memory, and storage for each component in a cluster. This enhancement simplifies the process of monitoring the resource usage rate of your cluster. - - List all import tasks - - Get an import task - - Create an import task - - Update an import task - - Upload a local file for an import task - - Preview data before starting an import task - - Get the role information for import tasks + To access these metric limits, navigate to the **Monitoring** page of your cluster, and then check the **Server** category under the **Metrics** tab. - For more information, refer to the [API documentation](https://docs.pingcap.com/tidbcloud/api/v1beta#tag/Import). + For more information, see [Metrics for TiDB Cloud Dedicated clusters](/tidb-cloud/built-in-monitoring.md#server). -## February 22, 2023 +## February 21, 2024 **General changes** -- Support using the [console audit logging](/tidb-cloud/tidb-cloud-console-auditing.md) feature to track various activities performed by members within your organization in the [TiDB Cloud console](https://tidbcloud.com/). - - The console audit logging feature is only visible to users with the `Owner` or `Audit Admin` role and is disabled by default. To enable it, click **Organization** > **Console Audit Logging** in the upper-right corner of the [TiDB Cloud console](https://tidbcloud.com/). - - By analyzing console audit logs, you can identify suspicious operations performed within your organization, thereby improving the security of your organization's resources and data. - - For more information, see [Console audit logging](/tidb-cloud/tidb-cloud-console-auditing.md). - -**CLI changes** - -- Add a new command [`ticloud cluster connect-info`](/tidb-cloud/ticloud-cluster-connect-info.md) for [TiDB Cloud CLI](/tidb-cloud/cli-reference.md). - - `ticloud cluster connect-info` is a command that allows you to get the connection string of a cluster. To use this command, [update `ticloud`](/tidb-cloud/ticloud-update.md) to v0.3.2 or a later version. +- Upgrade the TiDB version of [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters from [v6.6.0](https://docs.pingcap.com/tidb/v6.6/release-6.6.0) to [v7.1.3](https://docs.pingcap.com/tidb/v7.1/release-7.1.3). -## February 21, 2023 +## February 20, 2024 **General changes** -- Support using the AWS access keys of an IAM user to access your Amazon S3 bucket when importing data to TiDB Cloud. - - This method is simpler than using Role ARN. For more information, refer to [Configure Amazon S3 access](/tidb-cloud/config-s3-and-gcs-access.md#configure-amazon-s3-access). - -- Extend the [monitoring metrics retention period](/tidb-cloud/built-in-monitoring.md#metrics-retention-policy) from 2 days to a longer period: +- Support creating more TiDB Cloud nodes on Google Cloud. - - For Dedicated Tier clusters, you can view metrics data for the past 7 days. - - For Serverless Tier clusters, you can view metrics data for the past 3 days. - - By extending the metrics retention period, now you have access to more historical data. This helps you identify trends and patterns of the cluster for better decision-making and faster troubleshooting. - -**Console changes** + - By [configuring a regional CIDR size](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-cidr-for-a-region) of `/19` for Google Cloud, you can now create up to 124 TiDB Cloud nodes within any region of a project. + - If you want to create more than 124 nodes in any region of a project, you can contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md) for assistance in customizing an IP range size ranging from `/16` to `/18`. -- Release a new native web infrastructure on the Monitoring page of [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. - - With the new infrastructure, you can easily navigate through the Monitoring page and access the necessary information in a more intuitive and efficient manner. The new infrastructure also resolves many problems on UX, making the monitoring process a lot more user-friendly. - -## February 17, 2023 - -**CLI changes** - -- Add a new command [`ticloud connect`](/tidb-cloud/ticloud-connect.md) for [TiDB Cloud CLI](/tidb-cloud/cli-reference.md). - - `ticloud connect` is a command that allows you to connect to your TiDB Cloud cluster from your local machine without installing any SQL clients. After connecting to your TiDB Cloud cluster, you can execute SQL statements in the TiDB Cloud CLI. - -## February 14, 2023 +## January 23, 2024 **General changes** -- Support decreasing the number of TiKV and TiFlash nodes to scale in a TiDB [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) cluster. +- Add 32 vCPU as a node size option for TiDB, TiKV, and TiFlash. - You can decrease the node number [via the TiDB Cloud console](/tidb-cloud/scale-tidb-cluster.md#change-node-number) or [via the TiDB Cloud API (beta)](https://docs.pingcap.com/tidbcloud/api/v1beta#tag/Cluster/operation/UpdateCluster). + For each `32 vCPU, 128 GiB` TiKV node, the node storage ranges from 200 GiB to 6144 GiB. -**Console changes** - -- Introduce the **Monitoring** page for [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. - - The **Monitoring** page provides a range of metrics and data, such as the number of SQL statements executed per second, the average duration of queries, and the number of failed queries, which helps you better understand the overall performance of SQL statements in your Serverless Tier cluster. + It is recommended to use such nodes in the following scenarios: - For more information, see [TiDB Cloud built-in monitoring](/tidb-cloud/built-in-monitoring.md). - -## February 2, 2023 - -**CLI changes** + - High-workload production environments + - Extremely high performance -- Introduce the TiDB Cloud CLI client [`ticloud`](/tidb-cloud/cli-reference.md). - - Using `ticloud`, you can easily manage your TiDB Cloud resources from a terminal or other automatic workflows with a few lines of commands. Especially for GitHub Actions, we have provided [`setup-tidbcloud-cli`](https://github.com/marketplace/actions/set-up-tidbcloud-cli) for you to easily set up `ticloud`. - - For more information, see [TiDB Cloud CLI Quick Start](/tidb-cloud/get-started-with-cli.md) and [TiDB Cloud CLI Reference](/tidb-cloud/cli-reference.md). - -## January 18, 2023 - -**General changes** - -* Support [signing up](https://tidbcloud.com/free-trial) TiDB Cloud with a Microsoft account. - -## January 17, 2023 - -**General changes** - -- Upgrade the default TiDB version of new [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters from [v6.1.3](https://docs.pingcap.com/tidb/stable/release-6.1.3) to [v6.5.0](https://docs.pingcap.com/tidb/stable/release-6.5.0). - -- For new sign-up users, TiDB Cloud will automatically create a free [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster so that you can quickly start a data exploration journey with TiDB Cloud. - -- Support a new AWS region for [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters: `Seoul (ap-northeast-2)`. - - The following features are enabled for this region: - - - [Migrate MySQL-compatible databases to TiDB Cloud using Data Migration](/tidb-cloud/migrate-from-mysql-using-data-migration.md) - - [Stream data from TiDB Cloud to other data services using changefeed](/tidb-cloud/changefeed-overview.md) - - [Back up and restore TiDB cluster data](/tidb-cloud/backup-and-restore.md) - -## January 10, 2023 +## January 16, 2024 **General changes** -- Optimize the feature of importing data from local CSV files to TiDB to improve the user experience for [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. +- Enhance CIDR configuration for projects. - - To upload a CSV file, now you can simply drag and drop it to the upload area on the **Import** page. - - When creating an import task, if your target database or table does not exist, you can enter a name to let TiDB Cloud create it for you automatically. For the target table to be created, you can specify a primary key or select multiple fields to form a composite primary key. - - After the import is completed, you can explore your data with [AI-powered Chat2Query](/tidb-cloud/explore-data-with-chat2query.md) by clicking **Explore your data by Chat2Query** or clicking the target table name in the task list. + - You can directly set a region-level CIDR for each project. + - You can choose your CIDR configurations from a broader range of CIDR values. - For more information, see [Import local files to TiDB Cloud](/tidb-cloud/tidb-cloud-import-local-files.md). - -**Console changes** + Note: The previous global-level CIDR settings for projects are retired, but all existing regional CIDR in active state remain unaffected. There will be no impact on the network of existing clusters. -- Add the **Get Support** option for each cluster to simplify the process of requesting support for a specific cluster. + For more information, see [Set a CIDR for a region](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-cidr-for-a-region). - You can request support for a cluster in either of the following ways: +- TiDB Cloud Serverless users now have the capability to disable public endpoints for your clusters. - - On the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, click **...** in the row of your cluster and select **Get Support**. - - On your cluster overview page, click **...** in the upper-right corner and select **Get Support**. - -## January 5, 2023 - -**Console changes** + For more information, see [Disable a Public Endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md#disable-a-public-endpoint). -- Rename SQL Editor (beta) to Chat2Query (beta) for [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters and support generating SQL queries using AI. +- [Data Service (beta)](https://tidbcloud.com/console/data-service) supports configuring a custom domain to access endpoints in a Data App. - In Chat2Query, you can either let AI generate SQL queries automatically or write SQL queries manually, and run SQL queries against databases without a terminal. + By default, TiDB Cloud Data Service provides a domain `.data.tidbcloud.com` to access each Data App's endpoints. For enhanced personalization and flexibility, you can now configure a custom domain for your Data App instead of using the default domain. This feature enables you to use branded URLs for your database services and enhances security. - To access Chat2Query, go to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, click your cluster name, and then click **Chat2Query** in the left navigation pane. + For more information, see [Custom domain in Data Service](/tidb-cloud/data-service-custom-domain.md). -## January 4, 2023 +## January 3, 2024 **General changes** -- Support scaling up TiDB, TiKV, and TiFlash nodes by increasing the **Node Size(vCPU + RAM)** for TiDB Dedicated clusters hosted on AWS and created after December 31, 2022. +- Support [Organization SSO](https://tidbcloud.com/console/preferences/authentication) to streamline enterprise authentication processes. - You can increase the node size [using the TiDB Cloud console](/tidb-cloud/scale-tidb-cluster.md#change-vcpu-and-ram) or [using the TiDB Cloud API (beta)](https://docs.pingcap.com/tidbcloud/api/v1beta#tag/Cluster/operation/UpdateCluster). + With this feature, you can seamlessly integrate TiDB Cloud with any identity provider (IdP) using [Security Assertion Markup Language (SAML)](https://en.wikipedia.org/wiki/Security_Assertion_Markup_Language) or [OpenID Connect (OIDC)](https://openid.net/developers/how-connect-works/). -- Extend the metrics retention period on the [**Monitoring**](/tidb-cloud/built-in-monitoring.md) page to two days. + For more information, see [Organization SSO Authentication](/tidb-cloud/tidb-cloud-org-sso-authentication.md). - Now you have access to metrics data of the last two days, giving you more flexibility and visibility into your cluster performance and trends. +- Upgrade the default TiDB version of new [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters from [v7.1.1](https://docs.pingcap.com/tidb/v7.1/release-7.1.1) to [v7.5.0](https://docs.pingcap.com/tidb/v7.5/release-7.5.0). - This improvement comes at no additional cost and can be accessed on the **Diagnosis** tab of the [**Monitoring**](/tidb-cloud/built-in-monitoring.md) page for your cluster. This will help you identify and troubleshoot performance issues and monitor the overall health of your cluster more effectively. - -- Support customizing Grafana dashboard JSON for Prometheus integration. - - If you have [integrated TiDB Cloud with Prometheus](/tidb-cloud/monitor-prometheus-and-grafana-integration.md), you can now import a pre-built Grafana dashboard to monitor TiDB Cloud clusters and customize the dashboard to your needs. This feature enables easy and fast monitoring of your TiDB Cloud clusters and helps you identify any performance issues quickly. - - For more information, see [Use Grafana GUI dashboards to visualize the metrics](/tidb-cloud/monitor-prometheus-and-grafana-integration.md#step-3-use-grafana-gui-dashboards-to-visualize-the-metrics). - -- Upgrade the default TiDB version of all [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters from [v6.3.0](https://docs.pingcap.com/tidb/v6.3/release-6.3.0) to [v6.4.0](https://docs.pingcap.com/tidb/v6.4/release-6.4.0). The cold start issue after upgrading the default TiDB version of Serverless Tier clusters to v6.4.0 has been resolved. - -**Console changes** +- The dual region backup feature for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) is now in General Availability (GA). -- Simplify the display of the [**Clusters**](https://tidbcloud.com/console/clusters) page and the cluster overview page. + By using this feature, you can replicate backups across geographic regions within AWS or Google Cloud. This feature provides an additional layer of data protection and disaster recovery capabilities. - - You can click the cluster name on the [**Clusters**](https://tidbcloud.com/console/clusters) page to enter the cluster overview page and start operating the cluster. - - Remove the **Connection** and **Import** panes from the cluster overview page. You can click **Connect** in the upper-right corner to get the connection information and click **Import** in the left navigation pane to import data. + For more information, see [Dual region backup](/tidb-cloud/backup-and-restore.md#turn-on-dual-region-backup). diff --git a/tidb-cloud/tidb-cloud-sql-tuning-overview.md b/tidb-cloud/tidb-cloud-sql-tuning-overview.md index 1f7d130561cb8..0058552892dfc 100644 --- a/tidb-cloud/tidb-cloud-sql-tuning-overview.md +++ b/tidb-cloud/tidb-cloud-sql-tuning-overview.md @@ -27,13 +27,21 @@ The TiDB Cloud console provides a **[SQL Statement](/tidb-cloud/tune-performance Note that on this sub-tab, SQL queries with the same structure (even if the query parameters do not match) are grouped into the same SQL statement. For example, `SELECT * FROM employee WHERE id IN (1, 2, 3)` and `select * from EMPLOYEE where ID in (4, 5)` are both part of the same SQL statement `select * from employee where id in (...)`. -You can view some key information in **Statement**. +You can view some key information in **SQL Statement**. + +- **SQL Template**: including SQL digest, SQL template ID, the time range currently viewed, the number of execution plans, and the database where the execution takes place. + + ![Details0](/media/dashboard/dashboard-statement-detail0.png) -- SQL statement overview: including SQL digest, SQL template ID, the time range currently viewed, the number of execution plans, and the database where the execution takes place. - Execution plan list: if a SQL statement has more than one execution plan, the list is displayed. You can select different execution plans and the details of the selected execution plan are displayed at the bottom of the list. If there is only one execution plan, the list will not be displayed. -- Execution plan details: shows the details of the selected execution plan. It collects the execution plans of such SQL type and the corresponding execution time from several perspectives to help you get more information. See [Execution plan in details](https://docs.pingcap.com/tidb/stable/dashboard-statement-details#statement-execution-details-of-tidb-dashboard) (area 3 in the image below). -![Details](/media/dashboard/dashboard-statement-detail.png) + ![Details1](/media/dashboard/dashboard-statement-detail1.png) + +- Execution plan details: shows the details of the selected execution plan. It collects the execution plans of each SQL type and the corresponding execution time from several perspectives to help you get more information. See [Execution plans](https://docs.pingcap.com/tidb/stable/dashboard-statement-details#execution-plans). + + ![Details2](/media/dashboard/dashboard-statement-detail2.png) + +- Related slow query In addition to the information in the **Statement** dashboard, there are also some SQL best practices for TiDB Cloud as described in the following sections. diff --git a/tidb-cloud/tidb-cloud-sso-authentication.md b/tidb-cloud/tidb-cloud-sso-authentication.md index 4831f0494443b..9c5735aa7f266 100644 --- a/tidb-cloud/tidb-cloud-sso-authentication.md +++ b/tidb-cloud/tidb-cloud-sso-authentication.md @@ -1,9 +1,9 @@ --- -title: SSO Authentication +title: Basic SSO Authentication summary: Learn how to log in to the TiDB Cloud console via your Google, GitHub, or Microsoft account. --- -# SSO Authentication +# Basic SSO Authentication This document describes how to log in to the [TiDB Cloud console](https://tidbcloud.com/) via basic Single Sign-on (SSO) authentication, which is quick and convenient. diff --git a/tidb-cloud/tidb-cloud-support.md b/tidb-cloud/tidb-cloud-support.md index 295bed91b81b7..4283cf27e1d54 100644 --- a/tidb-cloud/tidb-cloud-support.md +++ b/tidb-cloud/tidb-cloud-support.md @@ -64,11 +64,11 @@ To check or upgrade your support plan, perform the following steps: 2. Choose your desired support plan. -
    +
    - To upgrade to **Standard**: + To upgrade to **Standard** or **Enterprise**: - 1. Click **Select Plan** in the **Standard** pane. A **Finish and Start Using Support** page is displayed. + 1. Click **Select Plan** in the **Standard** or **Enterprise** pane. A **Finish and Start Using Support** page is displayed. 2. Check the billing information in the lower-left corner of the page. 3. Add your payment information in the **Billing Profile** and **Credit Card** areas. @@ -76,14 +76,14 @@ To check or upgrade your support plan, perform the following steps: 4. Click **Confirm and Start Using Support** in the lower-right corner of the page. - After the payment is finished, you have upgraded your plan to **Standard**. + After the payment is finished, you have upgraded your plan to **Standard** or **Enterprise**.
    -
    +
    - To upgrade your plan to **Enterprise** or **Premium**: + To upgrade your plan to **Premium**: - 1. Click **Contact Sales** in the **Enterprise** or **Premium** pane. A **Contact Us** page is displayed. + 1. Click **Contact Sales** in the **Premium** pane. A **Contact Us** page is displayed. 2. Fill in and submit your contact information on the page. Then, the support team will contact you and help you with your subscription.
    diff --git a/tidb-cloud/tidb-cloud-tls-connect-to-dedicated.md b/tidb-cloud/tidb-cloud-tls-connect-to-dedicated.md index b123452b0c715..1af58081d15fd 100644 --- a/tidb-cloud/tidb-cloud-tls-connect-to-dedicated.md +++ b/tidb-cloud/tidb-cloud-tls-connect-to-dedicated.md @@ -1,52 +1,50 @@ --- -title: TLS Connections to TiDB Dedicated -summary: Introduce TLS connections in TiDB Dedicated. +title: TLS Connections to TiDB Cloud Dedicated +summary: Introduce TLS connections in TiDB Cloud Dedicated. aliases: ['/tidbcloud/tidb-cloud-tls-connect-to-dedicated-tier'] --- -# TLS Connections to TiDB Dedicated +# TLS Connections to TiDB Cloud Dedicated -On TiDB Cloud, establishing TLS connections is one of the basic security practices for connecting to TiDB Dedicated clusters. You can configure multiple TLS connections from your client, application, and development tools to your TiDB Dedicated cluster to protect data transmission security. For security reasons, TiDB Dedicated only supports TLS 1.2 and TLS 1.3, and does not support TLS 1.0 and TLS 1.1 versions. +On TiDB Cloud, establishing TLS connections is one of the basic security practices for connecting to TiDB Cloud Dedicated clusters. You can configure multiple TLS connections from your client, application, and development tools to your TiDB Cloud Dedicated cluster to protect data transmission security. For security reasons, TiDB Cloud Dedicated only supports TLS 1.2 and TLS 1.3, and does not support TLS 1.0 and TLS 1.1 versions. -To ensure data security, TiDB cluster CA for your TiDB Dedicated cluster is hosted on [AWS Certificate Manager (ACM)](https://aws.amazon.com/certificate-manager/), and TiDB cluster private keys are stored in AWS-managed hardware security modules (HSMs) that meet [FIPS 140-2 Level 3](https://csrc.nist.gov/projects/cryptographic-module-validation-program/Certificate/3139) security standards. +To ensure data security, TiDB cluster CA for your TiDB Cloud Dedicated cluster is hosted on [AWS Certificate Manager (ACM)](https://aws.amazon.com/certificate-manager/), and TiDB cluster private keys are stored in AWS-managed hardware security modules (HSMs) that meet [FIPS 140-2 Level 3](https://csrc.nist.gov/projects/cryptographic-module-validation-program/Certificate/3139) security standards. ## Prerequisites -- Log in to TiDB Cloud via [Password Authentication](/tidb-cloud/tidb-cloud-password-authentication.md) or [SSO Authentication](/tidb-cloud/tidb-cloud-sso-authentication.md), and then [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). +- Log in to TiDB Cloud via [Password Authentication](/tidb-cloud/tidb-cloud-password-authentication.md) or [SSO Authentication](/tidb-cloud/tidb-cloud-sso-authentication.md), and then [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). - Set a password to access your cluster in secure settings. - To do so, you can navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, click **...** in the row of your TiDB Dedicated cluster, and then select **Security Settings**. In security settings, you can click **Generate** to automatically generate a root password with a length of 16 characters, including numbers, uppercase and lowercase characters, and special characters. + To do so, you can navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, click **...** in the row of your TiDB Cloud Dedicated cluster, and then select **Password Settings**. In password settings, you can click **Auto-generate Password** to automatically generate a root password with a length of 16 characters, including numbers, uppercase and lowercase characters, and special characters. -## Secure connection to a TiDB Dedicated cluster +## Secure connection to a TiDB Cloud Dedicated cluster -In the [TiDB Cloud console](https://tidbcloud.com/), you can get examples of different connection methods and connect to your TiDB Dedicated cluster as follows: +In the [TiDB Cloud console](https://tidbcloud.com/), you can get examples of different connection methods and connect to your TiDB Cloud Dedicated cluster as follows: -1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your TiDB Dedicated cluster to go to its overview page. +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your TiDB Cloud Dedicated cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A dialog is displayed. -3. On the **Standard Connection** tab of this dialog, follow the three steps to set up the TLS connection. - - Step 1:Create traffic filter - - Step 2:Download TiDB cluster CA - - Step 3:Connect with an SQL client +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list. -4. Under **Step 1: Create traffic filter** in the dialog, configure the IP addresses that are allowed to access your cluster. For more information, see [Configure an IP access list in standard connection](/tidb-cloud/configure-ip-access-list.md#configure-an-ip-access-list-in-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** to configure it before your first connection. For more information, see [Configure an IP access list](/tidb-cloud/configure-ip-access-list.md). -5. Under **Step 2: Download TiDB cluster CA**, click **Download TiDB cluster CA** to download it locally for client TLS configuration. The TiDB cluster CA ensures that the TLS connection is secure and reliable. +4. Click **CA cert** to download CA cert for TLS connection to TiDB clusters. The CA cert supports TLS 1.2 version by default. > **Note:** > - > After downloading your TiDB Dedicated cluster CA, you can store it in the default storage path of your operating system, or specify another storage path. You need to replace the CA path in the code example with your own cluster CA path in the subsequent steps. + > - You can store the downloaded CA cert in the default storage path of your operating system, or specify another storage path. You need to replace the CA cert path in the code example with your own CA cert path in the subsequent steps. + > - TiDB Cloud Dedicated does not force clients to use TLS connections, and user-defined configuration of the [`require_secure_transport`](/system-variables.md#require_secure_transport-new-in-v610) variable is currently not supported on TiDB Cloud Dedicated. -6. Under **Step 3: Connect with an SQL client** in the dialog, click the tab of your preferred connection method, and then refer to the connection string and sample code on the tab to connect to your cluster. +5. Choose your preferred connection method, and then refer to the connection string and sample code on the tab to connect to your cluster. The following examples show the connection strings in MySQL, MyCLI, JDBC, Python, Go, and Node.js:
    -MySQL CLI client attempts to establish a TLS connection by default. When you connect to TiDB Dedicated clusters, you need to set `ssl-mode` and `ssl-ca`. +MySQL CLI client attempts to establish a TLS connection by default. When you connect to TiDB Cloud Dedicated clusters, you need to set `ssl-mode` and `ssl-ca`. ```shell mysql --connect-timeout 15 --ssl-mode=VERIFY_IDENTITY --ssl-ca=ca.pem --tls-version="TLSv1.2" -u root -h tidb.eqlfbdgthh8.clusters.staging.tidb-cloud.com -P 4000 -D test -p @@ -54,7 +52,7 @@ mysql --connect-timeout 15 --ssl-mode=VERIFY_IDENTITY --ssl-ca=ca.pem --tls-vers Parameter description: -- With `--ssl-mode=VERIFY_IDENTITY`, MySQL CLI client forces to enable TLS and validate TiDB Dedicated clusters. +- With `--ssl-mode=VERIFY_IDENTITY`, MySQL CLI client forces to enable TLS and validate TiDB Cloud Dedicated clusters. - Use `--ssl-ca=` to specify your local path of the downloaded TiDB cluster `ca.pem`. - Use `--tls-version=TLSv1.2` to restrict the versions of the TLS protocol. If you want to use TLS 1.3, you can set the version to `TLSv1.3`. @@ -62,7 +60,7 @@ Parameter description:
    -[MyCLI](https://www.mycli.net/) automatically enables TLS when using TLS related parameters. When you connect to TiDB Dedicated clusters, you need to set `ssl-ca` and `ssl-verify-server-cert`. +[MyCLI](https://www.mycli.net/) automatically enables TLS when using TLS related parameters. When you connect to TiDB Cloud Dedicated clusters, you need to set `ssl-ca` and `ssl-verify-server-cert`. ```shell mycli --ssl-ca=ca.pem --ssl-verify-server-cert -u root -h tidb.eqlfbdgthh8.clusters.staging.tidb-cloud.com -P 4000 -D test @@ -71,7 +69,7 @@ mycli --ssl-ca=ca.pem --ssl-verify-server-cert -u root -h tidb.eqlfbdgthh8.clust Parameter descriptions: - Use `--ssl-ca=` to specify your local path of the downloaded TiDB cluster `ca.pem`. -- With `--ssl-verify-server-cert` to validate TiDB Dedicated clusters. +- With `--ssl-verify-server-cert` to validate TiDB Cloud Dedicated clusters.
    @@ -116,7 +114,7 @@ class Main { Parameter description: -- Set `sslMode=VERIFY_IDENTITY` to enable TLS and validate TiDB Dedicated clusters. +- Set `sslMode=VERIFY_IDENTITY` to enable TLS and validate TiDB Cloud Dedicated clusters. - Set `enabledTLSProtocols=TLSv1.2` to restrict the versions of the TLS protocol. If you want to use TLS 1.3, you can set the version to `TLSv1.3`. - Set `trustCertificateKeyStoreUrl` to your custom truststore path. - Set `trustCertificateKeyStorePassword` to your truststore password. @@ -147,7 +145,7 @@ with connection: Parameter descriptions: -- Set `ssl_mode="VERIFY_IDENTITY"` to enable TLS and validate TiDB Dedicated clusters. +- Set `ssl_mode="VERIFY_IDENTITY"` to enable TLS and validate TiDB Cloud Dedicated clusters. - Use `ssl={"ca": ""}` to specify your local path of the downloaded TiDB cluster `ca.pem`.
    @@ -219,9 +217,9 @@ func main() { Parameter descriptions: -- Register `tls.Config` in the TLS connection configuration to enable TLS and validate TiDB Dedicated clusters. +- Register `tls.Config` in the TLS connection configuration to enable TLS and validate TiDB Cloud Dedicated clusters. - Set `MinVersion: tls.VersionTLS12` to restrict the versions of TLS protocol. -- Set `ServerName: ""` to verify TiDB Dedicated's hostname. +- Set `ServerName: ""` to verify TiDB Cloud Dedicated's hostname. - If you do not want to register a new TLS configuration, you can just set `tls=true` in the connection string.
    @@ -284,18 +282,18 @@ Parameter descriptions:
    -## Manage root certificates for TiDB Dedicated +## Manage root certificates for TiDB Cloud Dedicated -TiDB Dedicated uses certificates from [AWS Certificate Manager (ACM)](https://aws.amazon.com/certificate-manager/) as a Certificate Authority (CA) for TLS connections between clients and TiDB Dedicated clusters. Usually, the root certificates of ACM are stored securely in AWS-managed hardware security modules (HSMs) that meet [FIPS 140-2 Level 3](https://csrc.nist.gov/projects/cryptographic-module-validation-program/Certificate/3139) security standards. +TiDB Cloud Dedicated uses certificates from [AWS Certificate Manager (ACM)](https://aws.amazon.com/certificate-manager/) as a Certificate Authority (CA) for TLS connections between clients and TiDB Cloud Dedicated clusters. Usually, the root certificates of ACM are stored securely in AWS-managed hardware security modules (HSMs) that meet [FIPS 140-2 Level 3](https://csrc.nist.gov/projects/cryptographic-module-validation-program/Certificate/3139) security standards. ## FAQs -### Which TLS versions are supported to connect to my TiDB Dedicated cluster? +### Which TLS versions are supported to connect to my TiDB Cloud Dedicated cluster? -For security reasons, TiDB Dedicated only supports TLS 1.2 and TLS 1.3, and does not support TLS 1.0 and TLS 1.1 versions. See IETF [Deprecating TLS 1.0 and TLS 1.1](https://datatracker.ietf.org/doc/rfc8996/) for details. +For security reasons, TiDB Cloud Dedicated only supports TLS 1.2 and TLS 1.3, and does not support TLS 1.0 and TLS 1.1 versions. See IETF [Deprecating TLS 1.0 and TLS 1.1](https://datatracker.ietf.org/doc/rfc8996/) for details. -### Is two-way TLS authentication between my client and TiDB Dedicated supported? +### Is two-way TLS authentication between my client and TiDB Cloud Dedicated supported? No. -TiDB Dedicated only supports one-way TLS authentication, and does not support two-way TLS authentication currently. If you need two-way TLS authentication, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). +TiDB Cloud Dedicated only supports one-way TLS authentication, and does not support two-way TLS authentication currently. If you need two-way TLS authentication, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). diff --git a/tidb-cloud/tune-performance.md b/tidb-cloud/tune-performance.md index d7da01d3e9854..53083b9334821 100644 --- a/tidb-cloud/tune-performance.md +++ b/tidb-cloud/tune-performance.md @@ -17,7 +17,7 @@ TiDB Cloud provides [Slow Query](#slow-query), [Statement Analysis](#statement-a > **Note:** > -> Currently, **Key Visualizer** and **Index Insight (beta)** are unavailable for [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. +> Currently, **Key Visualizer** and **Index Insight (beta)** are unavailable for [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters. ## Slow Query @@ -57,7 +57,7 @@ For more information, see [Statement Execution Details in TiDB Dashboard](https: > **Note:** > -> Key Visualizer is only available for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. +> Key Visualizer is only available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters. To view the key analytics, perform the following steps: @@ -75,6 +75,6 @@ The Index Insight feature in TiDB Cloud provides powerful capabilities to optimi > **Note:** > -> Index Insight is currently in beta and only available for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. +> Index Insight is currently in beta and only available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters. For more information, see [Index Insight](/tidb-cloud/index-insight.md). diff --git a/tidb-cloud/use-chat2query-api.md b/tidb-cloud/use-chat2query-api.md index 05103954dbb0e..76045fbcd9ef4 100644 --- a/tidb-cloud/use-chat2query-api.md +++ b/tidb-cloud/use-chat2query-api.md @@ -5,99 +5,375 @@ summary: Learn how to use TiDB Cloud Chat2Query API to generate and execute SQL # Get Started with Chat2Query API -TiDB Cloud provides the Chat2Query API, a RESTful interface that allows you to generate and execute SQL statements using AI by providing instructions. Then, the API returns the query results for you. +TiDB Cloud provides the Chat2Query API, a RESTful interface that enables you to generate and execute SQL statements using AI by providing instructions. Then, the API returns the query results for you. Chat2Query API can only be accessed through HTTPS, ensuring that all data transmitted over the network is encrypted using TLS. > **Note:** > -> Chat2Query API is available for [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. To use the Chat2Query API on [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters, contact [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md). +> Chat2Query API is available for [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters. To use the Chat2Query API on [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters, contact [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md). ## Before you begin -Before using the Chat2Query API, make sure that you have created a TiDB cluster and enabled [AI to generate SQL queries](/tidb-cloud/explore-data-with-chat2query.md). If you do not have a TiDB cluster, follow the steps in [Create a TiDB Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) or [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md) to create one. +Before calling Chat2Query endpoints, you need to create a Chat2Query Data App and create an API key for the Data App. -## Step 1. Enable the Chat2Query API +### Create a Chat2Query Data App -To enable the Chat2Query API, perform the following steps: +To create a Data App for your project, perform the following steps: -1. Go to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. +1. On the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project, click **Create DataApp** in the left pane. The data app creation dialog is displayed. > **Tip:** > - > If you have multiple projects, you can click in the lower-left corner and switch to another project. + > If you are on the **SQL Editor** page of your cluster, you can also open the data app creation dialog by clicking **...** in the upper-right corner, choosing **Access Chat2Query via API**, and clicking **New Chat2Query Data App**. -2. Click your cluster name, and then click **Chat2Query** in the left navigation pane. -3. In the upper-right corner of Chat2Query, click **...** and select **Settings**. -4. Enable **DataAPI** and the Chat2Query Data App is created. +2. In the dialog, define a name for your Data App, choose the desired clusters as the data sources, and select **Chat2Query Data App** as the **Data App** type. Optionally, you can also write a description for the App. - > **Note:** - > - > After DataAPI is enabled for one TiDB cluster, all TiDB clusters in the same project can use the Chat2Query API. +3. Click **Create**. -5. Click the **Data Service** link in the message to access the Chat2Query API. + The newly created Chat2Query Data App is displayed in the left pane. Under this Data App, you can find a list of Chat2Query endpoints. - You can find that the **Chat2Query System** [Data App](/tidb-cloud/tidb-cloud-glossary.md#data-app) and its **Chat2Data** [endpoint](/tidb-cloud/tidb-cloud-glossary.md#endpoint) are displayed in the left pane. +### Create an API key -## Step 2. Create an API key +Before calling an endpoint, you need to create an API key for the Chat2Query Data App, which is used by the endpoint to access data in your TiDB Cloud clusters. -Before calling an endpoint, you need to create an API key. To create an API key for the Chat2Query Data App, perform the following steps: +To create an API key, perform the following steps: -1. In the left pane of [**Data Service**](https://tidbcloud.com/console/data-service), click the name of **Chat2Query System** to view its details. +1. In the left pane of [**Data Service**](https://tidbcloud.com/console/data-service), click your Chat2Query Data App to view its details on the right side. 2. In the **Authentication** area, click **Create API Key**. -3. In the **Create API Key** dialog box, enter a description and select a role for your API key. +3. In the **Create API Key** dialog, enter a description, and then select one of the following roles for your API key: + + - `Chat2Query Admin`: allows the API key to manage data summaries, generate SQL statements based on provided instructions, and execute any SQL statements. + - `Chat2Query Data Summary Management Role`: only allows the API key to generate and update data summaries. - The role is used to control whether the API key can read or write data to the clusters linked to the Data App. You can select the `ReadOnly` or `ReadAndWrite` role: + > **Tip:** + > + > For Chat2Query API, a data summary is an analysis result of your database by AI, including your database descriptions, table descriptions, and column descriptions. By generating a data summary of your database, you can get a more accurate response when generating SQL statements by providing instructions. - - `ReadOnly`: only allows the API key to read data, such as `SELECT`, `SHOW`, `USE`, `DESC`, and `EXPLAIN` statements. - - `ReadAndWrite`: allows the API key to read and write data. You can use this API key to execute all SQL statements, such as DML and DDL statements. + - `Chat2Query SQL ReadOnly`: only allows the API key to generate SQL statements based on provided instructions and execute `SELECT` SQL statements. + - `Chat2Query SQL ReadWrite`: allows the API key to generate SQL statements based on provided instructions and execute any SQL statements. -4. Click **Next**. The public key and private key are displayed. +4. By default, an API key never expires. If you prefer to set an expiration time for the key, click **Expires in**, select a time unit (`Minutes`, `Days`, or `Months`), and then fill in a desired number for the time unit. + +5. Click **Next**. The public key and private key are displayed. Make sure that you have copied and saved the private key in a secure location. After leaving this page, you will not be able to get the full private key again. -5. Click **Done**. +6. Click **Done**. + +## Call Chat2Query endpoints + +> **Note:** +> +> Each Chat2Query Data App has a rate limit of 100 requests per day. If you exceed the rate limit, the API returns a `429` error. For more quota, you can [submit a request](https://support.pingcap.com/hc/en-us/requests/new?ticket_form_id=7800003722519) to our support team. + +In each Chat2Query Data App, you can find the following endpoints: + +- Chat2Query v3 endpoints: the endpoints whose names starting with `/v3`, such as `/v3/dataSummaries` and `/v3/chat2data`(recommended) +- Chat2Query v2 endpoints: the endpoints whose names starting with `/v2`, such as `/v2/dataSummaries` and `/v2/chat2data` +- Chat2Query v1 endpoint: `/v1/chat2data`(deprecated) + +> **Tip:** +> +> Compared with `/v1/chat2data`, `/v3/chat2data` and `/v2/chat2data` requires you to analyze your database first by calling `/v3/dataSummaries` or `/v2/dataSummaries`. Consequently, the results returned by `/v3/chat2data` and `/v2/chat2data` are generally more accurate. + +### Get the code example of an endpoint + +TiDB Cloud provides code examples to help you quickly call Chat2Query endpoints. To get the code example of a Chat2Query endpoint, perform the following steps: + +1. In the left pane of the [**Data Service**](https://tidbcloud.com/console/data-service) page, click the name of a Chat2Query endpoint. + + The information for calling this endpoint is displayed on the right side, such as endpoint URL, code example, and request method. + +2. Click **Show Code Example**. + +3. In the displayed dialog box, select the cluster, database, and authentication method that you want to use to call the endpoint, and then copy the code example. + + > **Note:** + > + > For some of the endpoints such as `/v2/jobs/{job_id}`, you only need to select the authentication method. + +4. To call the endpoint, you can paste the example in your application, replace the parameters in the example with your own (such as replacing the `${PUBLIC_KEY}` and `${PRIVATE_KEY}` placeholders with your API key), and then run it. + +### Call Chat2Query v3 endpoints or v2 endpoints + +TiDB Cloud Data Service provides the following Chat2Query v3 endpoints and v2 endpoints: + +| Method | Endpoint | Description | +| ------ | -------- | ----------- | +| POST | `/v3/dataSummaries` | This endpoint generates a data summary for your database schema, table schema, and column schema by using artificial intelligence for analysis. | +| GET | `/v3/dataSummaries` | This endpoint retrieves all data summaries of your database. | +| GET | `/v3/dataSummaries/{data_summary_id}` | This endpoint retrieves a specific data summary. | +| PUT | `/v3/dataSummaries/{data_summary_id}` | This endpoint updates a specific data summary. | +| PUT | `/v3/dataSummaries/{data_summary_id}/tables/{table_name}` | This endpoint updates the description of a specific table in a specific data summary. | +| PUT | `/v3/dataSummaries/{data_summary_id}/tables/{table_name}/columns` | This endpoint updates the description of columns for a specific table in a specific data summary. | +| POST | `/v3/knowledgeBases` | This endpoint creates a new knowledge base. For more information about the usage of knowledge base related endpoints, see [Use knowledge bases](/tidb-cloud/use-chat2query-knowledge.md). | +| GET | `/v3/knowledgeBases` | This endpoint retrieves all knowledge bases. | +| GET | `/v3/knowledgeBases/{knowledge_base_id}` | This endpoint retrieves a specific knowledge base. | +| PUT | `/v3/knowledgeBases/{knowledge_base_id}` | This endpoint updates a specific knowledge base. | +| POST | `/v3/knowledgeBases/{knowledge_base_id}/data` | This endpoint adds data to a specific knowledge base. | +| GET | `/v3/knowledgeBases/{knowledge_base_id}/data` | This endpoint retrieves data from a specific knowledge base. | +| PUT | `/v3/knowledgeBases/{knowledge_base_id}/data/{knowledge_data_id}` | This endpoint updates specific data in a knowledge base. | +| DEL | `/v3/knowledgeBases/{knowledge_base_id}/data/{knowledge_data_id}` | This endpoint deletes specific data from a knowledge base. | +| POST | `/v3/sessions` | This endpoint creates a new session. For more information about the usage of session-related endpoints, see [Start multi-round Chat2Query](/tidb-cloud/use-chat2query-sessions.md). | +| GET | `/v3/sessions` | This endpoint retrieves a list of all sessions. | +| GET | `/v3/sessions/{session_id}` | This endpoint retrieves the details of a specific session. | +| PUT | `/v3/sessions/{session_id}` | This endpoint updates a specific session. | +| PUT | `/v3/sessions/{session_id}/reset` | This endpoint resets a specific session. | +| POST | `/v3/sessions/{session_id}/chat2data` | This endpoint generates and executes SQL statements within a specific session using artificial intelligence. For more information, see [Start multi-round Chat2Query by using sessions](/tidb-cloud/use-chat2query-sessions.md). | +| POST | `/v3/chat2data` | This endpoint enables you to generate and execute SQL statements using artificial intelligence by providing the data summary ID and instructions. | +| POST | `/v3/refineSql` | This endpoint refines existing SQL queries using artificial intelligence. | +| POST | `/v3/suggestQuestions` | This endpoint suggests questions based on the provided data summary. | +| POST | `/v2/dataSummaries` | This endpoint generates a data summary for your database schema, table schema, and column schema using artificial intelligence. | +| GET | `/v2/dataSummaries` | This endpoint retrieves all data summaries. | +| POST | `/v2/chat2data` | This endpoint enables you to generate and execute SQL statements using artificial intelligence by providing the data summary ID and instructions. | +| GET | `/v2/jobs/{job_id}` | This endpoint enables you to query the status of a specific data summary generation job. | + +The steps to call `/v3/chat2data` and `/v2/chat2data` are the same. The following sections take `/v3/chat2data` as an example to show how to call it. + +#### 1. Generate a data summary by calling `/v3/dataSummaries` + +Before calling `/v3/chat2data`, let AI analyze the database and generate a data summary first by calling `/v3/dataSummaries`, so `/v3/chat2data` can get a better performance in SQL generation later. + +The following is a code example of calling `/v3/dataSummaries` to analyze the `sp500insight` database and generate a data summary for the database: + +```bash +curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request POST 'https://.data.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/v3/dataSummaries'\ + --header 'content-type: application/json'\ + --data-raw '{ + "cluster_id": "10140100115280519574", + "database": "sp500insight", + "description": "Data summary for SP500 Insight", + "reuse": false +}' +``` + +In the preceding example, the request body is a JSON object with the following properties: + +- `cluster_id`: _string_. A unique identifier of the TiDB cluster. +- `database`: _string_. The name of the database. +- `description`: _string_. A description of the data summary. +- `reuse`: _boolean_. Specifies whether to reuse an existing data summary. If you set it to `true`, the API will reuse an existing data summary. If you set it to `false`, the API will generate a new data summary. + +An example response is as follows: + +```js +{ + "code": 200, + "msg": "", + "result": { + "data_summary_id": 304823, + "job_id": "fb99ef785da640ab87bf69afed60903d" + } +} +``` + +#### 2. Check the analysis status by calling `/v2/jobs/{job_id}` + +The `/v3/dataSummaries` API is asynchronous. For a database with a large dataset, it might take a few minutes to complete the database analysis and return the full data summary. + +To check the analysis status of your database, you can call the `/v2/jobs/{job_id}` endpoint as follows: + +```bash +curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request GET 'https://.data.dev.tidbcloud.com/api/v1beta/app/chat2query-`/endpoint/v2/jobs/{job_id}'\ + --header 'content-type: application/json' +``` + +An example response is as follows: + +```js +{ + "code": 200, + "msg": "", + "result": { + "ended_at": 1699518950, // A UNIX timestamp indicating when the job is finished + "job_id": "fb99ef785da640ab87bf69afed60903d", // ID of current job + "result": DataSummaryObject, // AI exploration information of the given database + "status": "done" // Status of the current job + } +} +``` + +If `"status"` is `"done"`, the full data summary is ready and you can now generate and execute SQL statements for this database by calling `/v3/chat2data`. Otherwise, you need to wait and check the analysis status later until it is done. -## Step 3. Call the Chat2Data endpoint +In the response, `DataSummaryObject` represents AI exploration information of the given database. The structure of `DataSummaryObject` is as follows: -In the left pane of the [**Data Service**](https://tidbcloud.com/console/data-service) page, click **Chat2Query** > **/chat2data** to view the endpoint details. The **Properties** of Chat2Data are displayed: +```js +{ + "cluster_id": "10140100115280519574", // The cluster ID + "data_summary_id": 304823, // The data summary ID + "database": "sp500insight", // The database name + "default": false, // Whether this data summary is the default one + "status": "done", // The status of the data summary + "description": { + "system": "Data source for financial analysis and decision-making in stock market", // The description of the data summary generated by AI + "user": "Data summary for SP500 Insight" // The description of the data summary provided by the user + }, + "keywords": ["User_Stock_Selection", "Index_Composition"], // Keywords of the data summary + "relationships": { + "companies": { + "referencing_table": "...", // The table that references the `companies` table + "referencing_table_column": "..." // The column that references the `companies` table + "referenced_table": "...", // The table that the `companies` table references + "referenced_table_column": "..." // The column that the `companies` table references + } + }, // Relationships between tables + "summary": "Financial data source for stock market analysis", // The summary of the data summary + "tables": { // Tables in the database + "companies": { + "name": "companies" // The table name + "description": "This table provides comprehensive...", // The description of the table + "columns": { + "city": { // Columns in the table + "name": "city" // The column name + "description": "The city where the company is headquartered.", // The description of the column + } + }, + }, + } +} +``` + +#### 3. Generate and execute SQL statements by calling `/v3/chat2data` + +When the data summary of a database is ready, you can call `/v3/chat2data` to generate and execute SQL statements by providing the cluster ID, database name, and your question. + +For example: + +```bash +curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request POST 'https://.data.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/v3/chat2data'\ + --header 'content-type: application/json'\ + --data-raw '{ + "cluster_id": "10140100115280519574", + "database": "sp500insight", + "question": "", + "sql_generate_mode": "direct" +}' +``` + +The request body is a JSON object with the following properties: -- **Endpoint Path**: (read-only) the path of the Chat2Data endpoint, which is `/chat2data`. +- `cluster_id`: _string_. A unique identifier of the TiDB cluster. +- `database`: _string_. The name of the database. +- `data_summary_id`: _integer_. The ID of the data summary used to generate SQL. This property only takes effect if `cluster_id` and `database` are not provided. If you specify both `cluster_id` and `database`, the API uses the default data summary of the database. +- `question`: _string_. A question in natural language describing the query you want. +- `sql_generate_mode`: _string_. The mode to generate SQL statements. The value can be `direct` or `auto_breakdown`. If you set it to `direct`, the API will generate SQL statements directly based on the `question` you provided. If you set it to `auto_breakdown`, the API will break down the `question` into multiple tasks and generate SQL statements for each task. -- **Endpoint URL**: (read-only) the URL of the Chat2Data endpoint, which is used to call the endpoint. For example, `https://.data.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/chat2data`. +An example response is as follows: -- **Request Method**: (read-only) the HTTP method of the Chat2Data endpoint, which is `POST`. +```js +{ + "code": 200, + "msg": "", + "result": { + "cluster_id": "10140100115280519574", + "database": "sp500insight", + "job_id": "20f7577088154d7889964f1a5b12cb26", + "session_id": 304832 + } +} +``` -- **Timeout(ms)**: the timeout for the Chat2Data endpoint, in milliseconds. +If you receive a response with the status code `400` as follows, it means that you need to wait a moment for the data summary to be ready. -- **Max Rows**: the maximum number of rows that the Chat2Data endpoint returns. +```js +{ + "code": 400, + "msg": "Data summary is not ready, please wait for a while and retry", + "result": {} +} +``` -TiDB Cloud generates code examples to help you call an endpoint. To get the examples and run the code, perform the following steps: +The `/v3/chat2data` API is asynchronous. You can check the job status by calling the `/v2/jobs/{job_id}` endpoint: -1. On the current **Chat2Data** page, click **Code Example** to the right of **Endpoint URL**. The **Code Example** dialog box is displayed. -2. In the dialog box, select the cluster and database that you want to use to call the endpoint, and then copy the code example. -3. Paste the code example in your application and run it. +```bash +curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request GET 'https://.data.dev.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/v2/jobs/{job_id}'\ + --header 'content-type: application/json' +``` - - Replace the `` and `` placeholders with your API key. - - Replace the `` placeholder with the instruction you want AI to generate and execute SQL statements. - - Replace the `` placeholder with the table name you want to query. If you do not specify a table name, AI will query all tables in the database. +An example response is as follows: + +```js +{ + "code": 200, + "msg": "", + "result": { + "ended_at": 1718785006, // A UNIX timestamp indicating when the job is finished + "job_id": "20f7577088154d7889964f1a5b12cb26", + "reason": "", // The reason for the job failure if the job fails + "result": { + "assumptions": [], + "chart_options": { // The generated chart options for the result + "chart_name": "Table", + "option": { + "columns": [ + "total_users" + ] + }, + "title": "Total Number of Users in the Database" + }, + "clarified_task": "Count the total number of users in the database.", // The clarified description of the task + "data": { // The data returned by the SQL statement + "columns": [ + { + "col": "total_users" + } + ], + "rows": [ + [ + "1" + ] + ] + }, + "description": "", + "sql": "SELECT COUNT(`user_id`) AS total_users FROM `users`;", // The generated SQL statement + "sql_error": null, // The error message of the SQL statement + "status": "done", // The status of the job + "task_id": "0", + "type": "data_retrieval" // The type of the job + }, + "status": "done" + } +} +``` + +### Call the Chat2Data v1 endpoint (deprecated) > **Note:** > -> Each Chat2Query Data App has a rate limit of 100 requests per day. If you exceed the rate limit, the API returns a `429` error. For more quota, you can [submit a request](https://support.pingcap.com/hc/en-us/requests/new?ticket_form_id=7800003722519) to our support team. +> The Chat2Data v1 endpoint is deprecated. It is recommended that you call Chat2Data v3 endpoints instead. + +TiDB Cloud Data Service provides the following Chat2Query v1 endpoint: + +| Method | Endpoint| Description | +| ---- | ---- |---- | +| POST | `/v1/chat2data` | This endpoint allows you to generate and execute SQL statements using artificial intelligence by providing the target database name and instructions. | + +You can call the `/v1/chat2data` endpoint directly to generate and execute SQL statements. Compared with `/v2/chat2data`, `/v1/chat2data` provides a faster response but lower performance. + +TiDB Cloud generates code examples to help you call an endpoint. To get the examples and run the code, see [Get the code example of an endpoint](#get-the-code-example-of-an-endpoint). + +When calling `/v1/chat2data`, you need to replace the following parameters: -The following code example is used to find the most popular GitHub repository from `sample_data.github_events` table: +- Replace the `${PUBLIC_KEY}` and `${PRIVATE_KEY}` placeholders with your API key. +- Replace the `` placeholder with the table name you want to query. If you do not specify a table name, AI will query all tables in the database. +- Replace the `` placeholder with the instruction you want AI to generate and execute SQL statements. + +> **Note:** +> +> Each Chat2Query Data App has a rate limit of 100 requests per day. If you exceed the rate limit, the API returns a `429` error. For more quota, you can [submit a request](https://support.pingcap.com/hc/en-us/requests/new?ticket_form_id=7800003722519) to our support team. +> An API Key with the role `Chat2Query Data Summary Management Role` cannot call the Chat2Data v1 endpoint. +The following code example is used to count how many users are in the `sp500insight.users` table: ```bash -curl --digest --user ':' \ - --request POST 'https://.data.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/chat2data' \ - --header 'content-type: application/json' \ - --data-raw '{ - "cluster_id": "12345678912345678960", - "database": "sample_data", - "tables": ["github_events"], - "instruction": "Find the most popular repo from GitHub events" - }' +curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request POST 'https://.data.dev.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/chat2data'\ + --header 'content-type: application/json'\ + --data-raw '{ + "cluster_id": "10939961583884005252", + "database": "sp500insight", + "tables": ["users"], + "instruction": "count the users" +}' ``` In the preceding example, the request body is a JSON object with the following properties: @@ -105,70 +381,67 @@ In the preceding example, the request body is a JSON object with the following p - `cluster_id`: _string_. A unique identifier of the TiDB cluster. - `database`: _string_. The name of the database. - `tables`: _array_. (optional) A list of table names to be queried. -- `instruction`: _string_. A natural language instruction describing the query you want. +- `instruction`: _string_. An instruction in natural language describing the query you want. The response is as follows: ```json { - "type": "chat2data_endpoint", - "data": { - "columns": [ - { - "col": "repo_name", - "data_type": "VARCHAR", - "nullable": false - }, - { - "col": "count", - "data_type": "BIGINT", - "nullable": false - } - ], - "rows": [ - { - "count": "2390", - "repo_name": "pytorch/pytorch" - } - ], - "result": { - "code": 200, - "message": "Query OK!", - "start_ms": 1678965476709, - "end_ms": 1678965476839, - "latency": "130ms", - "row_count": 1, - "row_affect": 0, - "limit": 50, - "sql": "SELECT sample_data.github_events.`repo_name`, COUNT(*) AS count FROM sample_data.github_events GROUP BY sample_data.github_events.`repo_name` ORDER BY count DESC LIMIT 1;", - "ai_latency": "30ms" - } + "type": "chat2data_endpoint", + "data": { + "columns": [ + { + "col": "COUNT(`user_id`)", + "data_type": "BIGINT", + "nullable": false + } + ], + "rows": [ + { + "COUNT(`user_id`)": "1" + } + ], + "result": { + "code": 200, + "message": "Query OK!", + "start_ms": 1699529488292, + "end_ms": 1699529491901, + "latency": "3.609656403s", + "row_count": 1, + "row_affect": 0, + "limit": 1000, + "sql": "SELECT COUNT(`user_id`) FROM `users`;", + "ai_latency": "3.054822491s" } + } +} ``` If your API call is not successful, you will receive a status code other than `200`. The following is an example of the `500` status code: ```json { - "type": "chat2data_endpoint", - "data": { - "columns": [], - "rows": [], - "result": { - "code": 500, - "message": "internal error! defaultPermissionHelper: rpc error: code = DeadlineExceeded desc = context deadline exceeded", - "start_ms": "", - "end_ms": "", - "latency": "", - "row_count": 0, - "row_affect": 0, - "limit": 0 - } + "type": "chat2data_endpoint", + "data": { + "columns": [], + "rows": [], + "result": { + "code": 500, + "message": "internal error! defaultPermissionHelper: rpc error: code = DeadlineExceeded desc = context deadline exceeded", + "start_ms": "", + "end_ms": "", + "latency": "", + "row_count": 0, + "row_affect": 0, + "limit": 0 } + } } ``` ## Learn more - [Manage an API key](/tidb-cloud/data-service-api-key.md) +- [Start Multi-round Chat2Query](/tidb-cloud/use-chat2query-sessions.md) +- [Use Knowledge Bases](/tidb-cloud/use-chat2query-knowledge.md) - [Response and Status Codes of Data Service](/tidb-cloud/data-service-response-and-status-code.md) diff --git a/tidb-cloud/use-chat2query-knowledge.md b/tidb-cloud/use-chat2query-knowledge.md new file mode 100644 index 0000000000000..99c0d614b5c29 --- /dev/null +++ b/tidb-cloud/use-chat2query-knowledge.md @@ -0,0 +1,218 @@ +--- +title: Use Knowledge Bases +summary: Learn how to improve your Chat2Query results by using Chat2Query knowledge base APIs. +--- + +# Use Knowledge Bases + +A knowledge base is a collection of structured data that can be used to enhance the SQL generation capabilities of Chat2Query. + +Starting from v3, the Chat2Query API enables you to add or modify knowledge bases by calling knowledge base related endpoints of your Chat2Query Data App. + +> **Note:** +> +> Knowledge base related endpoints are available for [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters by default. To use knowledge base related endpoints on [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters, contact [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md). + +## Before you begin + +Before creating a knowledge base for your database, make sure that you have the following: + +- A [Chat2Query Data App](/tidb-cloud/use-chat2query-api.md#create-a-chat2query-data-app) +- An [API key for the Chat2Query Data App](/tidb-cloud/use-chat2query-api.md#create-an-api-key) + +## Step 1. Create a knowledge base for the linked database + +> **Note:** +> +> The knowledge used by Chat2Query is **structured according to the database dimension**. You can connect multiple Chat2Query Data Apps to the same database, but each Chat2Query Data App can only use knowledge from a specific database it is linked to. + +In your Chat2Query Data App, you can create a knowledge base for a specific database by calling the `/v3/knowledgeBases` endpoint. After creation, you will get a `knowledge_base_id` for future knowledge management. + +The following is a general code example for calling this endpoint. + +> **Tip:** +> +> To get a specific code example for your endpoint, click the endpoint name in the left pane of your Data App, and then click **Show Code Example**. For more information, see [Get the example code of an endpoint](/tidb-cloud/use-chat2query-api.md#get-the-code-example-of-an-endpoint). + +```bash +curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request POST 'https://.data.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/v3/knowledgeBases'\ + --header 'content-type: application/json'\ + --data-raw '{ + "cluster_id": "", + "database": "", + "description": "" +}' +``` + +An example response is as follows: + +```json +{ + "code":200, + "msg":"", + "result": + { + "default":true, + "description":"", + "knowledge_base_id":2 + } +} +``` + +After getting the response, record the `knowledge_base_id` value in your response for later use. + +## Step 2. Choose a knowledge type + +The knowledge base of each database can contain multiple types of knowledge. Before adding knowledge to your knowledge base, you need to choose a knowledge type that best suits your use case. + +Currently, Chat2Query knowledge bases support the following knowledge types. Each type is specifically designed for different scenarios and has a unique knowledge structure. + +- [Few-shot example](#few-shot-example) +- [Term-sheet explanation](#term-sheet-explanation) +- [Instruction](#instruction) + +### Few-shot example + +Few-shot example refers to the Q&A learning samples provided to Chat2Query, which include sample questions and their corresponding answers. These examples help Chat2Query handle new tasks more effectively. + +> **Note:** +> +> Make sure the accuracy of newly added examples, because the quality of examples affects how well Chat2Query learns. Poor examples, such as mismatched questions and answers, can degrade the performance of Chat2Query on new tasks. + +#### Knowledge structure + +Each example consists of a sample question and its corresponding answer. + +For example: + +```json +{ + "question": "How many records are in the 'test' table?", + "answer": "SELECT COUNT(*) FROM `test`;" +} +``` + +#### Use cases + +Few-Shot examples can significantly improve the performance of Chat2Query in various scenarios, including but not limited to the following: + +1. **When dealing with rare or complex questions**: if Chat2Query encounters infrequent or complex questions, adding few-shot examples can enhance its understanding and improve the accuracy of the results. + +2. **When struggling with a certain type of question**: if Chat2Query frequently makes mistakes or has difficulty with specific questions, adding few-shot examples can help improve its performance on these questions. + +### Term-sheet explanation + +Term-sheet explanation refers to a comprehensive explanation of a specific term or a group of similar terms, helping Chat2Query understand the meaning and usage of these terms. + +> **Note:** +> +> Make sure the accuracy of newly added term explanations, because the quality of explanations affects how well Chat2Query learns. Incorrect interpretations do not improve Chat2Query results but also potentially lead to adverse effects. + +#### Knowledge structure + +Each explanation includes either a single term or a list of similar terms and their detailed descriptions. + +For example: + +```json +{ + "term": ["OSS"], + "description": "OSS Insight is a powerful tool that provides online data analysis for users based on nearly 6 billion rows of GitHub event data." +} +``` + +#### Use cases + +Term-sheet explanation is primarily used to improve Chat2Query's comprehension of user queries, especially in these situations: + +- **Dealing with industry-specific terminology or acronyms**: when your query contains industry-specific terminology or acronyms that might not be universally recognized, using a term-sheet explanation can help Chat2Query understand the meaning and usage of these terms. +- **Dealing with ambiguities in user queries**: when your query contains ambiguous concepts that is confusing, using a term-sheet explanation can help Chat2Query clarify these ambiguities. +- **Dealing with terms with various meanings**: when your query contains terms that carry different meanings in various contexts, using a term-sheet explanation can assist Chat2Query in discerning the correct interpretation. + +### Instruction + +Instruction is a piece of textual command. It is used to guide or control the behavior of Chat2Query, specifically instructing it on how to generate SQL according to specific requirements or conditions. + +> **Note:** +> +> - The instruction has a length limit of 512 characters. +> - Make sure to provide as clear and specific instructions as possible to ensure that Chat2Query can understand and execute the instructions effectively. + +#### Knowledge structure + +Instruction only includes a piece of textual command. + +For example: + +```json +{ + "instruction": "If the task requires calculating the sequential growth rate, use the LAG function with the OVER clause in SQL" +} +``` + +#### Use cases + +Instruction can be used in many scenarios to guide Chat2Query to output according to your requirements, including but not limited to the following: + +- **Limiting query scope**: if you want the SQL to consider only certain tables or columns, use an instruction to specify this. +- **Guiding SQL structure**: if you have specific requirements for the SQL structure, use an instruction to guide Chat2Query. + +## Step 3. Add knowledge to the newly created knowledge base + +To add new knowledge, you can call the `/v3/knowledgeBases/{knowledge_base_id}/data` endpoint. + +### Add a few-shot example type of knowledge + +For example, if you want Chat2Query to generate SQL statements of the count of rows in a table in a specific structure, you can add a few-shot example type of knowledge by calling `/v3/knowledgeBases/{knowledge_base_id}/data` as follows: + +```bash +curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request POST 'https://.data.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/v3/knowledgeBases//data'\ + --header 'content-type: application/json'\ + --data-raw '{ + "type": "few-shot", + "meta_data": {}, + "raw_data": { + "question": "How many records are in the 'test' table?", + "answer": "SELECT COUNT(*) FROM `test`;" + } +}' +``` + +In the preceding example code, `"type": "few-shot"` represents the few-shot example knowledge type. + +### Add a term-sheet explanation type of knowledge + +For example, if you want Chat2Query to comprehend the meaning of the term `OSS` using your provided explanation, you can add a term-sheet explanation type of knowledge by calling `/v3/knowledgeBases/{knowledge_base_id}/data` as follows: + +```bash +curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request POST 'https://.data.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/v3/knowledgeBases//data'\ + --header 'content-type: application/json'\ + --data-raw '{ + "type": "term-sheet", + "meta_data": {}, + "raw_data": { + "term": ["OSS"], + "description": "OSS Insight is a powerful tool that provides online data analysis for users based on nearly 6 billion rows of GitHub event data." + } +}' +``` + +In the preceding example code, `"type": "term-sheet"` represents the term-sheet explanation knowledge type. + +### Add an instruction type of knowledge + +For example, if you want Chat2Query to consistently use the `LAG` function with the `OVER` clause in SQL queries when dealing with questions about sequential growth rate calculation, you can add an instruction type of knowledge by calling `/v3/knowledgeBases/{knowledge_base_id}/data` as follows: + +```bash +curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request POST 'https://.data.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/v3/knowledgeBases//data'\ + --header 'content-type: application/json'\ + --data-raw '{ + "type": "instruction", + "meta_data": {}, + "raw_data": { + "instruction": "If the task requires calculating the sequential growth rate, use the LAG function with the OVER clause in SQL" + } +}' +``` + +In the preceding example code, `"type": "instruction"` represents the instruction knowledge type. \ No newline at end of file diff --git a/tidb-cloud/use-chat2query-sessions.md b/tidb-cloud/use-chat2query-sessions.md new file mode 100644 index 0000000000000..de8dbd8deb8b5 --- /dev/null +++ b/tidb-cloud/use-chat2query-sessions.md @@ -0,0 +1,101 @@ +--- +title: Start Multi-round Chat2Query +summary: Learn how to start multi-round chat by using Chat2Query session-related APIs. +--- + +# Start Multi-round Chat2Query + +Starting from v3, the Chat2Query API enables you to start multi-round chats by calling session related endpoints. You can use the `session_id` returned by the `/v3/chat2data` endpoint to continue your conversation in the next round. + +## Before you begin + +Before starting multi-round Chat2Query, make sure that you have the following: + +- A [Chat2Query Data App](/tidb-cloud/use-chat2query-api.md#create-a-chat2query-data-app). +- An [API key for the Chat2Query Data App](/tidb-cloud/use-chat2query-api.md#create-an-api-key). +- A [data summary for your target database](/tidb-cloud/use-chat2query-api.md#1-generate-a-data-summary-by-calling-v3datasummaries). + +## Step 1. Start a session + +To start a session, you can call the `/v3/sessions` endpoint of your Chat2Query Data App. + +The following is a general code example for calling this endpoint. + +> **Tip:** +> +> To get a specific code example for your endpoint, click the endpoint name in the left pane of your Data App, and then click **Show Code Example**. For more information, see [Get the example code of an endpoint](/tidb-cloud/use-chat2query-api.md#get-the-code-example-of-an-endpoint). + +```bash +curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request POST 'https://.data.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/v3/sessions'\ + --header 'content-type: application/json'\ + --data-raw '{ + "cluster_id": "10140100115280519574", + "database": "sp500insight", + "name": "" +}' +``` + +In the preceding code, the request body is a JSON object with the following properties: + +- `cluster_id`: _string_. A unique identifier of the TiDB cluster. +- `database`: _string_. The name of the database. +- `name`: _string_. The name of the session. + +An example response is as follows: + +```json +{ + "code": 200, + "msg": "", + "result": { + "messages": [], + "meta": { + "created_at": 1718948875, // A UNIX timestamp indicating when the session is created + "creator": "", // The creator of the session + "name": "", // The name of the session + "org_id": "1", // The organization ID + "updated_at": 1718948875 // A UNIX timestamp indicating when the session is updated + }, + "session_id": 305685 // The session ID + } +} +``` + +## Step 2. Call Chat2Data endpoints with the session + +After starting a session, you can call `/v3/sessions/{session_id}/chat2data` to continue your conversation in the next round. + +The following is a general code example: + +```bash +curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request POST 'https://eu-central-1.data.tidbcloud.com/api/v1beta/app/chat2query-YqAvnlRj/endpoint/v3/sessions/{session_id}/chat2data'\ + --header 'content-type: application/json'\ + --data-raw '{ + "question": "", + "feedback_answer_id": "", + "feedback_task_id": "", + "sql_generate_mode": "direct" +}' +``` + +In the preceding code, the request body is a JSON object with the following properties: + +- `question`: _string_. A question in natural language describing the query you want. +- `feedback_answer_id`: _string_. The feedback answer ID. This field is optional and is only used for feedback. +- `feedback_task_id`: _string_. The feedback task ID. This field is optional and is only used for feedback. +- `sql_generate_mode`: _string_. The mode to generate SQL statements. The value can be `direct` or `auto_breakdown`. If you set it to `direct`, the API will generate SQL statements directly based on the `question` you provided. If you set it to `auto_breakdown`, the API will break down the `question` into multiple tasks and generate SQL statements for each task. + +An example response is as follows: + +```json +{ + "code": 200, + "msg": "", + "result": { + "job_id": "d96b6fd23c5f445787eb5fd067c14c0b", + "session_id": 305685 + } +} +``` + +The response is similar to the response of the `/v3/chat2data` endpoint. You can check the job status by calling the `/v2/jobs/{job_id}` endpoint. For more information, see [Check the analysis status by calling `/v2/jobs/{job_id}`](/tidb-cloud/use-chat2query-api.md#2-check-the-analysis-status-by-calling-v2jobsjob_id). \ No newline at end of file diff --git a/tidb-cloud/use-htap-cluster.md b/tidb-cloud/use-htap-cluster.md index c39f00c8de520..6b818678b79a6 100644 --- a/tidb-cloud/use-htap-cluster.md +++ b/tidb-cloud/use-htap-cluster.md @@ -11,7 +11,7 @@ With TiDB Cloud, you can create an HTAP cluster easily by specifying one or more > **Note:** > -> TiFlash is always enabled for TiDB Serverless clusters. You cannot disable it. +> TiFlash is always enabled for TiDB Cloud Serverless clusters. You cannot disable it. TiKV data is not replicated to TiFlash by default. You can select which table to replicate to TiFlash using the following SQL statement: diff --git a/tidb-cloud/v6.5-performance-benchmarking-with-sysbench.md b/tidb-cloud/v6.5-performance-benchmarking-with-sysbench.md new file mode 100644 index 0000000000000..d8c4f4179fa6c --- /dev/null +++ b/tidb-cloud/v6.5-performance-benchmarking-with-sysbench.md @@ -0,0 +1,155 @@ +--- +title: TiDB Cloud Sysbench Performance Test Report for TiDB v6.5.6 +summary: Introduce the Sysbench performance test results for a TiDB Cloud Dedicated cluster with the TiDB version of v6.5.6. +--- + +# TiDB Cloud Sysbench Performance Test Report for TiDB v6.5.6 + +This document provides the Sysbench performance test steps and results for a TiDB Cloud Dedicated cluster with the TiDB version of v6.5.6. This report can also be used as a reference for the performance of TiDB Self-Managed v6.5.6 clusters. + +## Test overview + +This test aims at showing the Sysbench performance of TiDB v6.5.6 in the Online Transactional Processing (OLTP) scenario. + +## Test environment + +### TiDB cluster + +The test is conducted on a TiDB cluster with the following settings: + +- Cluster type: [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) +- Cluster version: v6.5.6 +- Cloud provider: AWS (us-west-2) +- Cluster configuration: + + | Node type | Node size | Node quantity | Node storage | + | :-------- | :-------------- | :------------ | :----------- | + | TiDB | 16 vCPU, 32 GiB | 2 | N/A | + | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | + +### Benchmark executor + +The benchmark executor sends SQL queries to the TiDB cluster. In this test, its hardware configuration is as follows: + +- Machine type: Amazon EC2 (us-west-2) +- Instance type: c6a.2xlarge +- Sysbench version: sysbench 1.0.20 (using bundled LuaJIT 2.1.0-beta2) + +## Test steps + +This section introduces how to perform the Sysbench performance test step by step. + +1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Cloud Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. + + For more information, see [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). + +2. On the benchmark executor, connect to the newly created cluster and create a database named `sbtest`. + + To connect to the cluster, see [Connect to TiDB Cloud Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). + + To create the `sbtest` database, execute the following SQL statement: + + ```sql + CREATE DATABASE sbtest; + ``` + +3. Load Sysbench data to the `sbtest` database. + + 1. The test in this document is implemented based on [sysbench](https://github.com/akopytov/sysbench). To install sysbench, see [Building and installing from source](https://github.com/akopytov/sysbench#building-and-installing-from-source). + + 2. Run the following `sysbench prepare` command to import 32 tables and 10,000,000 rows to the `sbtest` database. Replace `${HOST}`, `${PORT}`, `${THREAD}`, and `${PASSWORD}` with your actual values. + + ```shell + sysbench oltp_common \ + --threads=${THREAD} \ + --db-driver=mysql \ + --mysql-db=sbtest \ + --mysql-host=${HOST} \ + --mysql-port=${PORT} \ + --mysql-user=root \ + --mysql-password=${PASSWORD} \ + prepare --tables=32 --table-size=10000000 + ``` + +4. Run the following `sysbench run` command to conduct Sysbench performance tests on different workloads. This document conducts tests on five workloads: `oltp_point_select`, `oltp_read_write`, `oltp_update_non_index`, `oltp_update_index`, and `oltp_insert`. For each workload, this document conducts three tests with the `${THREAD}` value of `100`, `200`, and `400`. For each concurrency, the test takes 20 minutes. + + ```shell + sysbench ${WORKLOAD} run \ + --mysql-host=${HOST} \ + --mysql-port=${PORT} \ + --mysql-user=root \ + --db-driver=mysql \ + --mysql-db=sbtest \ + --threads=${THREAD} \ + --time=1200 \ + --report-interval=10 \ + --tables=32 \ + --table-size=10000000 \ + --mysql-ignore-errors=1062,2013,8028,9007 \ + --auto-inc=false \ + --mysql-password=${PASSWORD} + ``` + +## Test results + +This section introduces the Sysbench performance of v6.5.6 in the [test environment](#test-environment). + +### Point select performance + +The performance on the `oltp_point_select` workload is as follows: + +| Threads | TPS | 95% latency (ms) | +| :------ | :----- | :--------------- | +| 50 | 34125 | 2.03 | +| 100 | 64987 | 2.07 | +| 200 | 121656 | 2.14 | + +![Sysbench point select performance](/media/tidb-cloud/v6.5.6-oltp_select_point.png) + +### Read write performance + +The performance on the `oltp_read_write` workload is as follows: + +| Threads | TPS | 95% latency (ms) | +| :------ | :--- | :--------------- | +| 50 | 1232 | 46.6 | +| 100 | 2266 | 51.9 | +| 200 | 3578 | 81.5 | + +![Sysbench read write performance](/media/tidb-cloud/v6.5.6-oltp_read_write.png) + +### Update non-index performance + +The performance on the `oltp_update_non_index` workload is as follows: + +| Threads | TPS | 95% latency (ms) | +| :------ | :---- | :--------------- | +| 100 | 11016 | 11.0 | +| 200 | 20640 | 12.1 | +| 400 | 36830 | 13.5 | + +![Sysbench update non-index performance](/media/tidb-cloud/v6.5.6-oltp_update_non_index.png) + +### Update index performance + +The performance on the `oltp_update_index` workload is as follows: + +| Threads | TPS | 95% latency (ms) | +| :------ | :---- | :--------------- | +| 100 | 9270 | 14.0 | +| 200 | 14466 | 18.0 | +| 400 | 22194 | 24.8 | + +![Sysbench update index performance](/media/tidb-cloud/v6.5.6-oltp_update_index.png) + +### Insert performance + +The performance on the `oltp_insert` workload is as follows: + +| Threads | TPS | 95% latency (ms) | +| :------ | :---- | :--------------- | +| 100 | 16008 | 8.13 | +| 200 | 27143 | 10.1 | +| 400 | 40884 | 15.0 | + +![Sysbench insert performance](/media/tidb-cloud/v6.5.6-oltp_insert.png) diff --git a/tidb-cloud/v7.1.0-performance-benchmarking-with-tpcc.md b/tidb-cloud/v6.5-performance-benchmarking-with-tpcc.md similarity index 65% rename from tidb-cloud/v7.1.0-performance-benchmarking-with-tpcc.md rename to tidb-cloud/v6.5-performance-benchmarking-with-tpcc.md index 1d3fc2b4bcb94..3538c4001f194 100644 --- a/tidb-cloud/v7.1.0-performance-benchmarking-with-tpcc.md +++ b/tidb-cloud/v6.5-performance-benchmarking-with-tpcc.md @@ -1,15 +1,15 @@ --- -title: TiDB Cloud TPC-C Performance Test Report -summary: Introduce the TPC-C performance test results for TiDB Cloud. +title: TiDB Cloud TPC-C Performance Test Report for TiDB v6.5.6 +summary: Introduce the TPC-C performance test results for a TiDB Cloud Dedicated cluster with the TiDB version of v6.5.6. --- -# TiDB Cloud TPC-C Performance Test Report +# TiDB Cloud TPC-C Performance Test Report for TiDB v6.5.6 -This document provides the TPC-C performance test steps and results for a TiDB Dedicated cluster with the TiDB version of v7.1.0. This report can also be used as a reference for the performance of TiDB Self-Hosted v7.1.0 clusters. +This document provides the TPC-C performance test steps and results for a TiDB Cloud Dedicated cluster with the TiDB version of v6.5.6. This report can also be used as a reference for the performance of TiDB Self-Managed v6.5.6 clusters. ## Test overview -This test aims at showing the TPC-C performance of TiDB v7.1.0 in the Online Transactional Processing (OLTP) scenario. +This test aims at showing the TPC-C performance of TiDB v6.5.6 in the Online Transactional Processing (OLTP) scenario. ## Test environment @@ -17,15 +17,15 @@ This test aims at showing the TPC-C performance of TiDB v7.1.0 in the Online Tra The test is conducted on a TiDB cluster with the following settings: -- Cluster type: [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) -- Cluster version: v7.1.0 +- Cluster type: [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) +- Cluster version: v6.5.6 - Cloud provider: AWS (us-west-2) - Cluster configuration: - | Node type | Node size | Node quantity | Node storage | - |:----------|:----------|:----------|:----------| - | TiDB | 16 vCPU, 32 GiB | 2 | N/A | - | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | + | Node type | Node size | Node quantity | Node storage | + | :-------- | :-------------- | :------------ | :----------- | + | TiDB | 16 vCPU, 32 GiB | 2 | N/A | + | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | ### Benchmark executor @@ -38,13 +38,13 @@ The benchmark executor sends SQL queries to the TiDB cluster. In this test, its This section introduces how to perform the TPC-C performance test step by step. -1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. +1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Cloud Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. - For more information, see [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). + For more information, see [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). 2. On the benchmark executor, connect to the newly created cluster and create a database named `tpcc`. - To connect to the cluster, see [Connect to TiDB Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). + To connect to the cluster, see [Connect to TiDB Cloud Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). To create the `tpcc` database, execute the following SQL statement: @@ -88,7 +88,7 @@ This section introduces how to perform the TPC-C performance test step by step. SET tidb_index_serial_scan_concurrency=16; ``` -5. Run the following `go-tpc tpcc` command to conduct stress tests on the TiDB Dedicated cluster. For each concurrency, the test takes two hours. +5. Run the following `go-tpc tpcc` command to conduct stress tests on the TiDB Cloud Dedicated cluster. For each concurrency, the test takes two hours. ```shell go-tpc tpcc --host ${HOST} -P 4000 --warehouses 1000 run -D tpcc -T ${THREAD} --time 2h0m0s -p ${PASSWORD} --ignore-error @@ -100,12 +100,12 @@ This section introduces how to perform the TPC-C performance test step by step. ## Test results -The TPC-C performance of v7.1.0 in the [test environment](#test-environment) is as follows: +The TPC-C performance of v6.5.6 in the [test environment](#test-environment) is as follows: -| Threads | v7.1.0 tpmC | -|:--------|:----------| -| 50 | 36,159 | -| 100 | 66,742 | -| 200 | 94,406 | +| Threads | v6.5.6 tpmC | +| :------ | :---------- | +| 50 | 44183 | +| 100 | 74424 | +| 200 | 101545 | -![TPC-C](/media/tidb-cloud/v7.1.0-tpmC.png) +![TPC-C](/media/tidb-cloud/v6.5.6-tpmC.png) diff --git a/tidb-cloud/v7.1.0-performance-benchmarking-with-sysbench.md b/tidb-cloud/v7.1-performance-benchmarking-with-sysbench.md similarity index 62% rename from tidb-cloud/v7.1.0-performance-benchmarking-with-sysbench.md rename to tidb-cloud/v7.1-performance-benchmarking-with-sysbench.md index bbdc0d38335cc..66f28cc222005 100644 --- a/tidb-cloud/v7.1.0-performance-benchmarking-with-sysbench.md +++ b/tidb-cloud/v7.1-performance-benchmarking-with-sysbench.md @@ -1,15 +1,16 @@ --- -title: TiDB Cloud Sysbench Performance Test Report -summary: Introduce the Sysbench performance test results for TiDB Cloud. +title: TiDB Cloud Sysbench Performance Test Report for TiDB v7.1.3 +summary: Introduce the Sysbench performance test results for a TiDB Cloud Dedicated cluster with the TiDB version of v7.1.3. +aliases: ['/tidbcloud/v7.1.0-performance-benchmarking-with-sysbench'] --- -# TiDB Cloud Sysbench Performance Test Report +# TiDB Cloud Sysbench Performance Test Report for TiDB v7.1.3 -This document provides the Sysbench performance test steps and results for a TiDB Dedicated cluster with the TiDB version of v7.1.0. This report can also be used as a reference for the performance of TiDB v7.1.0 clusters. +This document provides the Sysbench performance test steps and results for a TiDB Cloud Dedicated cluster with the TiDB version of v7.1.3. This report can also be used as a reference for the performance of TiDB Self-Managed v7.1.3 clusters. ## Test overview -This test aims at showing the Sysbench performance of TiDB v7.1.0 in the Online Transactional Processing (OLTP) scenario. +This test aims at showing the Sysbench performance of TiDB v7.1.3 in the Online Transactional Processing (OLTP) scenario. ## Test environment @@ -17,15 +18,15 @@ This test aims at showing the Sysbench performance of TiDB v7.1.0 in the Online The test is conducted on a TiDB cluster with the following settings: -- Cluster type: [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) -- Cluster version: v7.1.0 +- Cluster type: [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) +- Cluster version: v7.1.3 - Cloud provider: AWS (us-west-2) - Cluster configuration: - | Node type | Node size | Node quantity | Node storage | - |:----------|:----------|:----------|:----------| - | TiDB | 16 vCPU, 32 GiB | 2 | N/A | - | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | + | Node type | Node size | Node quantity | Node storage | + | :-------- | :-------------- | :------------ | :----------- | + | TiDB | 16 vCPU, 32 GiB | 2 | N/A | + | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | ### Parameter configuration @@ -59,13 +60,13 @@ The benchmark executor sends SQL queries to the TiDB cluster. In this test, its This section introduces how to perform the Sysbench performance test step by step. -1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. +1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Cloud Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. - For more information, see [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). + For more information, see [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). 2. On the benchmark executor, connect to the newly created cluster and create a database named `sbtest`. - To connect to the cluster, see [Connect to TiDB Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). + To connect to the cluster, see [Connect to TiDB Cloud Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). To create the `sbtest` database, execute the following SQL statement: @@ -112,64 +113,64 @@ This section introduces how to perform the Sysbench performance test step by ste ## Test results -This section introduces the Sysbench performance of v7.1.0 in the [test environment](#test-environment). +This section introduces the Sysbench performance of v7.1.3 in the [test environment](#test-environment). ### Point select performance The performance on the `oltp_point_select` workload is as follows: -| Threads | TPS | 95% latency (ms)| -|:--------|:----------|:----------| -| 100 | 56,039 | 2.34 | -| 200 | 95,908 | 2.78 | -| 400 | 111,810 | 5.57 | +| Threads | TPS | 95% latency (ms) | +| :------ | :----- | :--------------- | +| 50 | 35309 | 1.93 | +| 100 | 64853 | 2.00 | +| 200 | 118462 | 2.22 | -![Sysbench point select performance](/media/tidb-cloud/v7.1.0-oltp_select_point.png) +![Sysbench point select performance](/media/tidb-cloud/v7.1.3-oltp_select_point.png) ### Read write performance The performance on the `oltp_read_write` workload is as follows: -| Threads | TPS | 95% latency (ms)| -|:--------|:----------|:----------| -| 100 | 1,789 | 66.8 | -| 200 | 2,842 | 97.6 | -| 400 | 3,090 | 191 | +| Threads | TPS | 95% latency (ms) | +| :------ | :--- | :--------------- | +| 50 | 1218 | 48.3 | +| 100 | 2235 | 53.9 | +| 200 | 3380 | 87.6 | -![Sysbench read write performance](/media/tidb-cloud/v.7.1.0-oltp_read_write.png) +![Sysbench read write performance](/media/tidb-cloud/v7.1.3-oltp_read_write.png) ### Update non-index performance The performance on the `oltp_update_non_index` workload is as follows: -| Threads | TPS | 95% latency (ms)| -|:--------|:----------|:----------| -| 100 | 7,944 | 16.7 | -| 200 | 13,844 | 19.0 | -| 400 | 29,063 | 20.4 | +| Threads | TPS | 95% latency (ms) | +| :------ | :---- | :--------------- | +| 100 | 10928 | 11.7 | +| 200 | 19985 | 12.8 | +| 400 | 35621 | 14.7 | -![Sysbench update non-index performance](/media/tidb-cloud/v7.1.0-oltp_update_non_index.png) +![Sysbench update non-index performance](/media/tidb-cloud/v7.1.3-oltp_update_non_index.png) ### Update index performance The performance on the `oltp_update_index` workload is as follows: -| Threads | TPS | 95% latency (ms)| -|:--------|:----------|:----------| -| 100 | 6,389 | 20 | -| 200 | 12,583 | 22.3 | -| 400 | 22,393 | 25.7 | +| Threads | TPS | 95% latency (ms) | +| :------ | :---- | :--------------- | +| 100 | 8854 | 14.7 | +| 200 | 14414 | 18.6 | +| 400 | 21997 | 25.3 | -![Sysbench update index performance](/media/tidb-cloud/v7.1.0-oltp_update_index.png) +![Sysbench update index performance](/media/tidb-cloud/v7.1.3-oltp_update_index.png) ### Insert performance The performance on the `oltp_insert` workload is as follows: -| Threads | TPS | 95% latency (ms)| -|:--------|:----------|:----------| -| 100 | 7,671 | 17.3 | -| 200 | 13,584 | 19.7 | -| 400 | 31,252 | 20 | +| Threads | TPS | 95% latency (ms) | +| :------ | :---- | :--------------- | +| 100 | 15575 | 8.13 | +| 200 | 25078 | 11.0 | +| 400 | 38436 | 15.6 | -![Sysbench insert performance](/media/tidb-cloud/v7.1.0-oltp_insert.png) +![Sysbench insert performance](/media/tidb-cloud/v7.1.3-oltp_insert.png) diff --git a/tidb-cloud/v7.1-performance-benchmarking-with-tpcc.md b/tidb-cloud/v7.1-performance-benchmarking-with-tpcc.md new file mode 100644 index 0000000000000..4cd0f9f38710e --- /dev/null +++ b/tidb-cloud/v7.1-performance-benchmarking-with-tpcc.md @@ -0,0 +1,112 @@ +--- +title: TiDB Cloud TPC-C Performance Test Report for TiDB v7.1.3 +summary: Introduce the TPC-C performance test results for a TiDB Cloud Dedicated cluster with the TiDB version of v7.1.3. +aliases: ['/tidbcloud/v7.1.0-performance-benchmarking-with-tpcc'] +--- + +# TiDB Cloud TPC-C Performance Test Report for TiDB v7.1.3 + +This document provides the TPC-C performance test steps and results for a TiDB Cloud Dedicated cluster with the TiDB version of v7.1.3. This report can also be used as a reference for the performance of TiDB Self-Managed v7.1.3 clusters. + +## Test overview + +This test aims at showing the TPC-C performance of TiDB v7.1.3 in the Online Transactional Processing (OLTP) scenario. + +## Test environment + +### TiDB cluster + +The test is conducted on a TiDB cluster with the following settings: + +- Cluster type: [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) +- Cluster version: v7.1.3 +- Cloud provider: AWS (us-west-2) +- Cluster configuration: + + | Node type | Node size | Node quantity | Node storage | + | :-------- | :-------------- | :------------ | :----------- | + | TiDB | 16 vCPU, 32 GiB | 2 | N/A | + | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | + +### Benchmark executor + +The benchmark executor sends SQL queries to the TiDB cluster. In this test, its hardware configuration is as follows: + +- Machine type: Amazon EC2 (us-west-2) +- Instance type: c6a.2xlarge + +## Test steps + +This section introduces how to perform the TPC-C performance test step by step. + +1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Cloud Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. + + For more information, see [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). + +2. On the benchmark executor, connect to the newly created cluster and create a database named `tpcc`. + + To connect to the cluster, see [Connect to TiDB Cloud Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). + + To create the `tpcc` database, execute the following SQL statement: + + ```sql + CREATE DATABASE tpcc; + ``` + +3. Load TPC-C data to the `tpcc` database. + + 1. The test in this document is implemented based on [go-tpc](https://github.com/pingcap/go-tpc). You can download the test program using the following command: + + ```shell + curl --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/pingcap/go-tpc/master/install.sh | sh + ``` + + 2. Run the following `go-tpc tpcc` command to import 1,000 warehouses to the `tpcc` database. Replace `${HOST}`, `${THREAD}`, and `${PASSWORD}` with your actual values. This document conducts three tests with the `${THREAD}` value of `50`, `100`, and `200`. + + ```shell + go-tpc tpcc --host ${HOST} --warehouses 1000 prepare -P 4000 -D tpcc -T ${THREAD} --time 2h0m0s -p ${PASSWORD} --ignore-error + ``` + +4. To ensure that the TiDB optimizer can generate the optimal execution plan, execute the following SQL statements to collect statistics before conducting the TPC-C test: + + ```sql + ANALYZE TABLE customer; + ANALYZE TABLE district; + ANALYZE TABLE history; + ANALYZE TABLE item; + ANALYZE TABLE new_order; + ANALYZE TABLE order_line; + ANALYZE TABLE orders; + ANALYZE TABLE stock; + ANALYZE TABLE warehouse; + ``` + + To accelerate the collection of statistics, execute the following SQL statements before collecting: + + ```sql + SET tidb_build_stats_concurrency=16; + SET tidb_distsql_scan_concurrency=16; + SET tidb_index_serial_scan_concurrency=16; + ``` + +5. Run the following `go-tpc tpcc` command to conduct stress tests on the TiDB Cloud Dedicated cluster. For each concurrency, the test takes two hours. + + ```shell + go-tpc tpcc --host ${HOST} -P 4000 --warehouses 1000 run -D tpcc -T ${THREAD} --time 2h0m0s -p ${PASSWORD} --ignore-error + ``` + +6. Extract the tpmC data of `NEW_ORDER` from the result. + + TPC-C uses tpmC (transactions per minute) to measure the maximum qualified throughput (MQTh, Max Qualified Throughput). The transactions are the NewOrder transactions and the final unit of measure is the number of new orders processed per minute. + +## Test results + +The TPC-C performance of v7.1.3 in the [test environment](#test-environment) is as follows: + +| Threads | v7.1.3 tpmC | +| :------ | :---------- | +| 50 | 42839 | +| 100 | 72895 | +| 200 | 97924 | + +![TPC-C](/media/tidb-cloud/v7.1.3-tpmC.png) diff --git a/tidb-cloud/v7.5-performance-benchmarking-with-sysbench.md b/tidb-cloud/v7.5-performance-benchmarking-with-sysbench.md new file mode 100644 index 0000000000000..3ceb83626726d --- /dev/null +++ b/tidb-cloud/v7.5-performance-benchmarking-with-sysbench.md @@ -0,0 +1,176 @@ +--- +title: TiDB Cloud Sysbench Performance Test Report for TiDB v7.5.0 +summary: Introduce the Sysbench performance test results for a TiDB Cloud Dedicated cluster with the TiDB version of v7.5.0. +aliases: ['/tidbcloud/v7.5.0-performance-benchmarking-with-sysbench'] +--- + +# TiDB Cloud Sysbench Performance Test Report for TiDB v7.5.0 + +This document provides the Sysbench performance test steps and results for a TiDB Cloud Dedicated cluster with the TiDB version of v7.5.0. This report can also be used as a reference for the performance of TiDB Self-Managed v7.5.0 clusters. + +## Test overview + +This test aims at showing the Sysbench performance of TiDB v7.5.0 in the Online Transactional Processing (OLTP) scenario. + +## Test environment + +### TiDB cluster + +The test is conducted on a TiDB cluster with the following settings: + +- Cluster type: [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) +- Cluster version: v7.5.0 +- Cloud provider: AWS (us-west-2) +- Cluster configuration: + + | Node type | Node size | Node quantity | Node storage | + | :-------- | :-------------- | :------------ | :----------- | + | TiDB | 16 vCPU, 32 GiB | 2 | N/A | + | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | + +### Parameter configuration + +> **Note:** +> +> For TiDB Cloud, to modify the TiKV parameters of your cluster, you can contact [PingCAP Support](/tidb-cloud/tidb-cloud-support.md) for help. + +The TiKV parameter [`prefill-for-recycle`](https://docs.pingcap.com/tidb/stable/tikv-configuration-file#prefill-for-recycle-new-in-v700) can make log recycling effective immediately after initialization. This document conducts tests based on different workloads with the following `prefill-for-recycle` configuration: + +- For the `oltp_point_select` workload, use the default value of the [`prefill-for-recycle`](https://docs.pingcap.com/tidb/stable/tikv-configuration-file#prefill-for-recycle-new-in-v700) parameter: + + ```yaml + raft-engine.prefill-for-recycle = false + ``` + +- For `oltp_insert`, `oltp_read_write` , `oltp_update_index`, and `oltp_update_non_index` workloads, enable the [`prefill-for-recycle`](https://docs.pingcap.com/tidb/stable/tikv-configuration-file#prefill-for-recycle-new-in-v700) parameter: + + ```yaml + raft-engine.prefill-for-recycle = true + ``` + +### Benchmark executor + +The benchmark executor sends SQL queries to the TiDB cluster. In this test, its hardware configuration is as follows: + +- Machine type: Amazon EC2 (us-west-2) +- Instance type: c6a.2xlarge +- Sysbench version: sysbench 1.0.20 (using bundled LuaJIT 2.1.0-beta2) + +## Test steps + +This section introduces how to perform the Sysbench performance test step by step. + +1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Cloud Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. + + For more information, see [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). + +2. On the benchmark executor, connect to the newly created cluster and create a database named `sbtest`. + + To connect to the cluster, see [Connect to TiDB Cloud Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). + + To create the `sbtest` database, execute the following SQL statement: + + ```sql + CREATE DATABASE sbtest; + ``` + +3. Load Sysbench data to the `sbtest` database. + + 1. The test in this document is implemented based on [sysbench](https://github.com/akopytov/sysbench). To install sysbench, see [Building and installing from source](https://github.com/akopytov/sysbench#building-and-installing-from-source). + + 2. Run the following `sysbench prepare` command to import 32 tables and 10,000,000 rows to the `sbtest` database. Replace `${HOST}`, `${PORT}`, `${THREAD}`, and `${PASSWORD}` with your actual values. + + ```shell + sysbench oltp_common \ + --threads=${THREAD} \ + --db-driver=mysql \ + --mysql-db=sbtest \ + --mysql-host=${HOST} \ + --mysql-port=${PORT} \ + --mysql-user=root \ + --mysql-password=${PASSWORD} \ + prepare --tables=32 --table-size=10000000 + ``` + +4. Run the following `sysbench run` command to conduct Sysbench performance tests on different workloads. This document conducts tests on five workloads: `oltp_point_select`, `oltp_read_write`, `oltp_update_non_index`, `oltp_update_index`, and `oltp_insert`. For each workload, this document conducts three tests with the `${THREAD}` value of `100`, `200`, and `400`. For each concurrency, the test takes 20 minutes. + + ```shell + sysbench ${WORKLOAD} run \ + --mysql-host=${HOST} \ + --mysql-port=${PORT} \ + --mysql-user=root \ + --db-driver=mysql \ + --mysql-db=sbtest \ + --threads=${THREAD} \ + --time=1200 \ + --report-interval=10 \ + --tables=32 \ + --table-size=10000000 \ + --mysql-ignore-errors=1062,2013,8028,9007 \ + --auto-inc=false \ + --mysql-password=${PASSWORD} + ``` + +## Test results + +This section introduces the Sysbench performance of v7.5.0 in the [test environment](#test-environment). + +### Point select performance + +The performance on the `oltp_point_select` workload is as follows: + +| Threads | TPS | 95% latency (ms) | +| :------ | :------ | :--------------- | +| 50 | 33,344 | 1.96 | +| 100 | 64,810 | 2.03 | +| 200 | 118,651 | 2.22 | + +![Sysbench point select performance](/media/tidb-cloud/v7.5.0-oltp_point_select.png) + +### Read write performance + +The performance on the `oltp_read_write` workload is as follows: + +| Threads | TPS | 95% latency (ms) | +| :------ | :---- | :--------------- | +| 50 | 1,181 | 49.2 | +| 100 | 2,162 | 54.8 | +| 200 | 3,169 | 92.4 | + +![Sysbench read write performance](/media/tidb-cloud/v7.5.0-oltp_read_write.png) + +### Update non-index performance + +The performance on the `oltp_update_non_index` workload is as follows: + +| Threads | TPS | 95% latency (ms) | +| :------ | :----- | :--------------- | +| 100 | 10,567 | 11.7 | +| 200 | 20,223 | 13.0 | +| 400 | 34,011 | 14.7 | + +![Sysbench update non-index performance](/media/tidb-cloud/v7.5.0-oltp_update_non_index.png) + +### Update index performance + +The performance on the `oltp_update_index` workload is as follows: + +| Threads | TPS | 95% latency (ms) | +| :------ | :----- | :--------------- | +| 100 | 8,896 | 14.7 | +| 200 | 13,718 | 19.0 | +| 400 | 20,377 | 26.9 | + +![Sysbench update index performance](/media/tidb-cloud/v7.5.0-oltp_update_index.png) + +### Insert performance + +The performance on the `oltp_insert` workload is as follows: + +| Threads | TPS | 95% latency (ms) | +| :------ | :----- | :--------------- | +| 100 | 15,132 | 8.58 | +| 200 | 24,756 | 10.8 | +| 400 | 37,247 | 16.4 | + +![Sysbench insert performance](/media/tidb-cloud/v7.5.0-oltp_insert.png) diff --git a/tidb-cloud/v7.5-performance-benchmarking-with-tpcc.md b/tidb-cloud/v7.5-performance-benchmarking-with-tpcc.md new file mode 100644 index 0000000000000..63ec70b77b0f1 --- /dev/null +++ b/tidb-cloud/v7.5-performance-benchmarking-with-tpcc.md @@ -0,0 +1,112 @@ +--- +title: TiDB Cloud TPC-C Performance Test Report for TiDB v7.5.0 +summary: Introduce the TPC-C performance test results for a TiDB Cloud Dedicated cluster with the TiDB version of v7.5.0. +aliases: ['/tidbcloud/v7.5.0-performance-benchmarking-with-tpcc'] +--- + +# TiDB Cloud TPC-C Performance Test Report for TiDB v7.5.0 + +This document provides the TPC-C performance test steps and results for a TiDB Cloud Dedicated cluster with the TiDB version of v7.5.0. This report can also be used as a reference for the performance of TiDB Self-Managed v7.5.0 clusters. + +## Test overview + +This test aims at showing the TPC-C performance of TiDB v7.5.0 in the Online Transactional Processing (OLTP) scenario. + +## Test environment + +### TiDB cluster + +The test is conducted on a TiDB cluster with the following settings: + +- Cluster type: [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) +- Cluster version: v7.5.0 +- Cloud provider: AWS (us-west-2) +- Cluster configuration: + + | Node type | Node size | Node quantity | Node storage | + | :-------- | :-------------- | :------------ | :----------- | + | TiDB | 16 vCPU, 32 GiB | 2 | N/A | + | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | + +### Benchmark executor + +The benchmark executor sends SQL queries to the TiDB cluster. In this test, its hardware configuration is as follows: + +- Machine type: Amazon EC2 (us-west-2) +- Instance type: c6a.2xlarge + +## Test steps + +This section introduces how to perform the TPC-C performance test step by step. + +1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Cloud Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. + + For more information, see [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). + +2. On the benchmark executor, connect to the newly created cluster and create a database named `tpcc`. + + To connect to the cluster, see [Connect to TiDB Cloud Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). + + To create the `tpcc` database, execute the following SQL statement: + + ```sql + CREATE DATABASE tpcc; + ``` + +3. Load TPC-C data to the `tpcc` database. + + 1. The test in this document is implemented based on [go-tpc](https://github.com/pingcap/go-tpc). You can download the test program using the following command: + + ```shell + curl --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/pingcap/go-tpc/master/install.sh | sh + ``` + + 2. Run the following `go-tpc tpcc` command to import 1,000 warehouses to the `tpcc` database. Replace `${HOST}`, `${THREAD}`, and `${PASSWORD}` with your actual values. This document conducts three tests with the `${THREAD}` value of `50`, `100`, and `200`. + + ```shell + go-tpc tpcc --host ${HOST} --warehouses 1000 prepare -P 4000 -D tpcc -T ${THREAD} --time 2h0m0s -p ${PASSWORD} --ignore-error + ``` + +4. To ensure that the TiDB optimizer can generate the optimal execution plan, execute the following SQL statements to collect statistics before conducting the TPC-C test: + + ```sql + ANALYZE TABLE customer; + ANALYZE TABLE district; + ANALYZE TABLE history; + ANALYZE TABLE item; + ANALYZE TABLE new_order; + ANALYZE TABLE order_line; + ANALYZE TABLE orders; + ANALYZE TABLE stock; + ANALYZE TABLE warehouse; + ``` + + To accelerate the collection of statistics, execute the following SQL statements before collecting: + + ```sql + SET tidb_build_stats_concurrency=16; + SET tidb_distsql_scan_concurrency=16; + SET tidb_index_serial_scan_concurrency=16; + ``` + +5. Run the following `go-tpc tpcc` command to conduct stress tests on the TiDB Cloud Dedicated cluster. For each concurrency, the test takes two hours. + + ```shell + go-tpc tpcc --host ${HOST} -P 4000 --warehouses 1000 run -D tpcc -T ${THREAD} --time 2h0m0s -p ${PASSWORD} --ignore-error + ``` + +6. Extract the tpmC data of `NEW_ORDER` from the result. + + TPC-C uses tpmC (transactions per minute) to measure the maximum qualified throughput (MQTh, Max Qualified Throughput). The transactions are the NewOrder transactions and the final unit of measure is the number of new orders processed per minute. + +## Test results + +The TPC-C performance of v7.5.0 in the [test environment](#test-environment) is as follows: + +| Threads | v7.5.0 tpmC | +| :------ | :---------- | +| 50 | 41,426 | +| 100 | 71,499 | +| 200 | 97,389 | + +![TPC-C](/media/tidb-cloud/v7.5.0_tpcc.png) diff --git a/tidb-cloud/v8.1-performance-benchmarking-with-sysbench.md b/tidb-cloud/v8.1-performance-benchmarking-with-sysbench.md new file mode 100644 index 0000000000000..b3bd825524fa2 --- /dev/null +++ b/tidb-cloud/v8.1-performance-benchmarking-with-sysbench.md @@ -0,0 +1,181 @@ +--- +title: TiDB Cloud Sysbench Performance Test Report for TiDB v8.1.0 +summary: Introduce the Sysbench performance test results for a TiDB Cloud Dedicated cluster with the TiDB version of v8.1.0. +--- + +# TiDB Cloud Sysbench Performance Test Report for TiDB v8.1.0 + +This document provides the Sysbench performance test steps and results for a TiDB Cloud Dedicated cluster with the TiDB version of v8.1.0. This report can also be used as a reference for the performance of TiDB Self-Managed v8.1.0 clusters. + +## Test overview + +This test aims at showing the Sysbench performance of TiDB v8.1.0 in the Online Transactional Processing (OLTP) scenario. + +## Test environment + +### TiDB cluster + +The test is conducted on a TiDB cluster with the following settings: + +- Cluster type: [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) +- Cluster version: v8.1.0 +- Cloud provider: AWS (us-west-2) +- Cluster configuration: + + | Node type | Node size | Node quantity | Node storage | + |:----------|:----------|:----------|:----------| + | TiDB | 16 vCPU, 32 GiB | 2 | N/A | + | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | + +### Parameter configuration + +The system variable [`tidb_session_plan_cache_size`](https://docs.pingcap.com/tidb/stable/system-variables#tidb_session_plan_cache_size-new-in-v710) controls the maximum number of plans that can be cached. The default value is `100`. For each workload, this document conducts tests with `tidb_session_plan_cache_size` set to `1000`: + +```sql +SET GLOBAL tidb_session_plan_cache_size = 1000; +``` + +> **Note:** +> +> For TiDB Cloud, to modify the TiKV parameters of your cluster, you can contact [PingCAP Support](/tidb-cloud/tidb-cloud-support.md) for help. + +The TiKV parameter [`prefill-for-recycle`](https://docs.pingcap.com/tidb/stable/tikv-configuration-file#prefill-for-recycle-new-in-v700) can make log recycling effective immediately after initialization. This document conducts tests based on different workloads with the following `prefill-for-recycle` configuration: + +- For the `oltp_point_select` workload, use the default value of the [`prefill-for-recycle`](https://docs.pingcap.com/tidb/stable/tikv-configuration-file#prefill-for-recycle-new-in-v700) parameter: + + ```yaml + raft-engine.prefill-for-recycle = false + ``` + +- For `oltp_insert`, `oltp_read_write`, `oltp_update_index`, and `oltp_update_non_index` workloads, enable the [`prefill-for-recycle`](https://docs.pingcap.com/tidb/stable/tikv-configuration-file#prefill-for-recycle-new-in-v700) parameter: + + ```yaml + raft-engine.prefill-for-recycle = true + ``` + +### Benchmark executor + +The benchmark executor sends SQL queries to the TiDB cluster. In this test, its hardware configuration is as follows: + +- Machine type: Amazon EC2 (us-west-2) +- Instance type: c6a.2xlarge +- Sysbench version: sysbench 1.0.20 (using bundled LuaJIT 2.1.0-beta2) + +## Test steps + +This section introduces how to perform the Sysbench performance test step by step. + +1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Cloud Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. + + For more information, see [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). + +2. On the benchmark executor, connect to the newly created cluster and create a database named `sbtest`. + + To connect to the cluster, see [Connect to TiDB Cloud Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). + + To create the `sbtest` database, execute the following SQL statement: + + ```sql + CREATE DATABASE sbtest; + ``` + +3. Load Sysbench data to the `sbtest` database. + + 1. The test in this document is implemented based on [sysbench](https://github.com/akopytov/sysbench). To install sysbench, see [Building and installing from source](https://github.com/akopytov/sysbench#building-and-installing-from-source). + + 2. Run the following `sysbench prepare` command to import 32 tables and 10,000,000 rows to the `sbtest` database. Replace `${HOST}`, `${PORT}`, `${THREAD}`, and `${PASSWORD}` with your actual values. + + ```shell + sysbench oltp_common \ + --threads=${THREAD} \ + --db-driver=mysql \ + --mysql-db=sbtest \ + --mysql-host=${HOST} \ + --mysql-port=${PORT} \ + --mysql-user=root \ + --mysql-password=${PASSWORD} \ + prepare --tables=32 --table-size=10000000 + ``` + +4. Run the following `sysbench run` command to conduct Sysbench performance tests on different workloads. This document conducts tests on five workloads: `oltp_point_select`, `oltp_read_write`, `oltp_update_non_index`, `oltp_update_index`, and `oltp_insert`. For each workload, this document conducts three tests with different values for the `${THREAD}` variable. For `oltp_point_select` and `oltp_read_write`, the values are `50`, `100`, and `200`. For other workloads, the values are `100`, `200`, and `400`. For each concurrency, the test takes 20 minutes. + + ```shell + sysbench ${WORKLOAD} run \ + --mysql-host=${HOST} \ + --mysql-port=${PORT} \ + --mysql-user=root \ + --db-driver=mysql \ + --mysql-db=sbtest \ + --threads=${THREAD} \ + --time=1200 \ + --report-interval=10 \ + --tables=32 \ + --table-size=10000000 \ + --mysql-ignore-errors=1062,2013,8028,9007 \ + --auto-inc=false \ + --mysql-password=${PASSWORD} + ``` + +## Test results + +This section introduces the Sysbench performance of v8.1.0 in the [test environment](#test-environment). + +### Point select performance + +The performance on the `oltp_point_select` workload is as follows: + +| Threads | TPS | 95% latency (ms)| +|:--------|:----------|:----------| +| 50 | 32,741 | 1.96 | +| 100 | 62,545 | 2.03 | +| 200 | 111,470 | 2.48 | + +![Sysbench point select performance](/media/tidb-cloud/v8.1.0_oltp_point_select.png) + +### Read write performance + +The performance on the `oltp_read_write` workload is as follows: + +| Threads | TPS | 95% latency (ms)| +|:--------|:----------|:----------| +| 50 | 1,232 | 46.6 | +| 100 | 2,341 | 51 | +| 200 | 3,240 | 109 | + +![Sysbench read write performance](/media/tidb-cloud/v8.1.0_oltp_read_write.png) + +### Update non-index performance + +The performance on the `oltp_update_non_index` workload is as follows: + +| Threads | TPS | 95% latency (ms)| +|:--------|:----------|:----------| +| 100 | 14,000 | 9.39 | +| 200 | 25,215 | 10.5 | +| 400 | 42,550 | 12.8 | + +![Sysbench update non-index performance](/media/tidb-cloud/v8.1.0_oltp_update_non_index.png) + +### Update index performance + +The performance on the `oltp_update_index` workload is as follows: + +| Threads | TPS | 95% latency (ms)| +|:--------|:----------|:----------| +| 100 | 11,188 | 11.7 | +| 200 | 17,805 | 14.7 | +| 400 | 24,575 | 23.5 | + +![Sysbench update index performance](/media/tidb-cloud/v8.1.0_oltp_update_index.png) + +### Insert performance + +The performance on the `oltp_insert` workload is as follows: + +| Threads | TPS | 95% latency (ms)| +|:--------|:----------|:----------| +| 100 | 18,339 | 7.3| +| 200 | 29,387 | 9.73 | +| 400 | 42,712 | 14.2 | + +![Sysbench insert performance](/media/tidb-cloud/v8.1.0_oltp_insert.png) diff --git a/tidb-cloud/v8.1-performance-benchmarking-with-tpcc.md b/tidb-cloud/v8.1-performance-benchmarking-with-tpcc.md new file mode 100644 index 0000000000000..8aad399883823 --- /dev/null +++ b/tidb-cloud/v8.1-performance-benchmarking-with-tpcc.md @@ -0,0 +1,123 @@ +--- +title: TiDB Cloud TPC-C Performance Test Report for TiDB v8.1.0 +summary: Introduce the TPC-C performance test results for a TiDB Cloud Dedicated cluster with the TiDB version of v8.1.0. +--- + +# TiDB Cloud TPC-C Performance Test Report for TiDB v8.1.0 + +This document provides the TPC-C performance test steps and results for a TiDB Cloud Dedicated cluster with the TiDB version of v8.1.0. This report can also be used as a reference for the performance of TiDB Self-Managed v8.1.0 clusters. + +## Test overview + +This test aims at showing the TPC-C performance of TiDB v8.1.0 in the Online Transactional Processing (OLTP) scenario. + +## Test environment + +### TiDB cluster + +The test is conducted on a TiDB cluster with the following settings: + +- Cluster type: [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) +- Cluster version: v8.1.0 +- Cloud provider: AWS (us-west-2) +- Cluster configuration: + + | Node type | Node size | Node quantity | Node storage | + |:----------|:----------|:----------|:----------| + | TiDB | 16 vCPU, 32 GiB | 2 | N/A | + | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | + +### Parameter configuration + +> **Note:** +> +> For TiDB Cloud, to modify the TiKV parameters of your cluster, you can contact [PingCAP Support](/tidb-cloud/tidb-cloud-support.md) for help. + +The TiKV parameter [`prefill-for-recycle`](https://docs.pingcap.com/tidb/stable/tikv-configuration-file#prefill-for-recycle-new-in-v700) can make log recycling effective immediately after initialization. This document conducts tests with `prefill-for-recycle` enabled: + +```yaml +raft-engine.prefill-for-recycle = true +``` + +### Benchmark executor + +The benchmark executor sends SQL queries to the TiDB cluster. In this test, its hardware configuration is as follows: + +- Machine type: Amazon EC2 (us-west-2) +- Instance type: c6a.2xlarge + +## Test steps + +This section introduces how to perform the TPC-C performance test step by step. + +1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Cloud Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. + + For more information, see [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). + +2. On the benchmark executor, connect to the newly created cluster and create a database named `tpcc`. + + To connect to the cluster, see [Connect to TiDB Cloud Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). + + To create the `tpcc` database, execute the following SQL statement: + + ```sql + CREATE DATABASE tpcc; + ``` + +3. Load TPC-C data to the `tpcc` database. + + 1. The test in this document is implemented based on [go-tpc](https://github.com/pingcap/go-tpc). You can download the test program using the following command: + + ```shell + curl --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/pingcap/go-tpc/master/install.sh | sh + ``` + + 2. Run the following `go-tpc tpcc` command to import 1,000 warehouses to the `tpcc` database. Replace `${HOST}`, `${THREAD}`, and `${PASSWORD}` with your actual values. This document conducts three tests with the `${THREAD}` value of `50`, `100`, and `200`. + + ```shell + go-tpc tpcc --host ${HOST} --warehouses 1000 prepare -P 4000 -D tpcc -T ${THREAD} --time 2h0m0s -p ${PASSWORD} --ignore-error + ``` + +4. To ensure that the TiDB optimizer can generate the optimal execution plan, execute the following SQL statements to collect statistics before conducting the TPC-C test: + + ```sql + ANALYZE TABLE customer; + ANALYZE TABLE district; + ANALYZE TABLE history; + ANALYZE TABLE item; + ANALYZE TABLE new_order; + ANALYZE TABLE order_line; + ANALYZE TABLE orders; + ANALYZE TABLE stock; + ANALYZE TABLE warehouse; + ``` + + To accelerate the collection of statistics, execute the following SQL statements before collecting: + + ```sql + SET tidb_build_stats_concurrency=16; + SET tidb_distsql_scan_concurrency=16; + SET tidb_index_serial_scan_concurrency=16; + ``` + +5. Run the following `go-tpc tpcc` command to conduct stress tests on the TiDB Cloud Dedicated cluster. For each concurrency, the test takes two hours. + + ```shell + go-tpc tpcc --host ${HOST} -P 4000 --warehouses 1000 run -D tpcc -T ${THREAD} --time 2h0m0s -p ${PASSWORD} --ignore-error + ``` + +6. Extract the tpmC data of `NEW_ORDER` from the result. + + TPC-C uses tpmC (transactions per minute) to measure the maximum qualified throughput (MQTh, Max Qualified Throughput). The transactions are the NewOrder transactions and the final unit of measure is the number of new orders processed per minute. + +## Test results + +The TPC-C performance of v8.1.0 in the [test environment](#test-environment) is as follows: + +| Threads | v8.1.0 tpmC | +|:--------|:----------| +| 50 | 43,660 | +| 100 | 75,495 | +| 200 | 102,013 | + +![TPC-C](/media/tidb-cloud/v8.1.0_tpcc.png) diff --git a/tidb-cloud/vector-search-changelogs.md b/tidb-cloud/vector-search-changelogs.md new file mode 100644 index 0000000000000..7333993d71ea0 --- /dev/null +++ b/tidb-cloud/vector-search-changelogs.md @@ -0,0 +1,14 @@ +--- +title: Vector Search Changelogs +summary: Learn about the new features, compatibility changes, improvements, and bug fixes for the TiDB Vector Search feature. +--- + +# Vector Search Changelogs + +## June 25, 2024 + +- TiDB Vector Search (beta) is now available for TiDB Cloud Serverless clusters in all regions for all users. + +## April 1, 2024 + +- TiDB Vector Search (beta) is now available for TiDB Cloud Serverless clusters in EU regions for invited users. diff --git a/tidb-cloud/vector-search-data-types.md b/tidb-cloud/vector-search-data-types.md new file mode 100644 index 0000000000000..0c390ab6f03e5 --- /dev/null +++ b/tidb-cloud/vector-search-data-types.md @@ -0,0 +1,246 @@ +--- +title: Vector Data Types +summary: Learn about the Vector data types in TiDB. +--- + +# Vector Data Types + +A vector is a sequence of floating-point numbers, such as `[0.3, 0.5, -0.1, ...]`. TiDB offers Vector data types, specifically optimized for efficiently storing and querying vector embeddings widely used in AI applications. + +The following Vector data types are currently available: + +- `VECTOR`: A sequence of single-precision floating-point numbers with any dimension. +- `VECTOR(D)`: A sequence of single-precision floating-point numbers with a fixed dimension `D`. + +Using vector data types provides the following advantages over using the [`JSON`](/data-type-json.md) type: + +- Vector index support: You can build a [vector search index](/tidb-cloud/vector-search-index.md) to speed up vector searching. +- Dimension enforcement: You can specify a dimension to forbid inserting vectors with different dimensions. +- Optimized storage format: Vector data types are optimized for handling vector data, offering better space efficiency and performance compared to `JSON` types. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## Syntax + +You can use a string in the following syntax to represent a Vector value: + +```sql +'[, , ...]' +``` + +Example: + +```sql +CREATE TABLE vector_table ( + id INT PRIMARY KEY, + embedding VECTOR(3) +); + +INSERT INTO vector_table VALUES (1, '[0.3, 0.5, -0.1]'); + +INSERT INTO vector_table VALUES (2, NULL); +``` + +Inserting vector values with invalid syntax will result in an error: + +```sql +[tidb]> INSERT INTO vector_table VALUES (3, '[5, ]'); +ERROR 1105 (HY000): Invalid vector text: [5, ] +``` + +In the following example, because dimension `3` is enforced for the `embedding` column when the table is created, inserting a vector with a different dimension will result in an error: + +```sql +[tidb]> INSERT INTO vector_table VALUES (4, '[0.3, 0.5]'); +ERROR 1105 (HY000): vector has 2 dimensions, does not fit VECTOR(3) +``` + +For available functions and operators over the vector data types, see [Vector Functions and Operators](/tidb-cloud/vector-search-functions-and-operators.md). + +For more information about building and using a vector search index, see [Vector Search Index](/tidb-cloud/vector-search-index.md). + +## Store vectors with different dimensions + +You can store vectors with different dimensions in the same column by omitting the dimension parameter in the `VECTOR` type: + +```sql +CREATE TABLE vector_table ( + id INT PRIMARY KEY, + embedding VECTOR +); + +INSERT INTO vector_table VALUES (1, '[0.3, 0.5, -0.1]'); -- 3 dimensions vector, OK +INSERT INTO vector_table VALUES (2, '[0.3, 0.5]'); -- 2 dimensions vector, OK +``` + +However, note that you cannot build a [vector search index](/tidb-cloud/vector-search-index.md) for this column, as vector distances can be only calculated between vectors with the same dimensions. + +## Comparison + +You can compare vector data types using [comparison operators](/functions-and-operators/operators.md) such as `=`, `!=`, `<`, `>`, `<=`, and `>=`. For a complete list of comparison operators and functions for vector data types, see [Vector Functions and Operators](/tidb-cloud/vector-search-functions-and-operators.md). + +Vector data types are compared element-wise numerically. For example: + +- `[1] < [12]` +- `[1,2,3] < [1,2,5]` +- `[1,2,3] = [1,2,3]` +- `[2,2,3] > [1,2,3]` + +Two vectors with different dimensions are compared using lexicographical comparison, with the following rules: + +- Two vectors are compared element by element from the start, and each element is compared numerically. +- The first mismatching element determines which vector is lexicographically _less_ or _greater_ than the other. +- If one vector is a prefix of another, the shorter vector is lexicographically _less_ than the other. For example, `[1,2,3] < [1,2,3,0]`. +- Vectors of the same length with identical elements are lexicographically _equal_. +- An empty vector is lexicographically _less_ than any non-empty vector. For example, `[] < [1]`. +- Two empty vectors are lexicographically _equal_. + +When comparing vector constants, consider performing an [explicit cast](#cast) from string to vector to avoid comparisons based on string values: + +```sql +-- Because string is given, TiDB is comparing strings: +[tidb]> SELECT '[12.0]' < '[4.0]'; ++--------------------+ +| '[12.0]' < '[4.0]' | ++--------------------+ +| 1 | ++--------------------+ +1 row in set (0.01 sec) + +-- Cast to vector explicitly to compare by vectors: +[tidb]> SELECT VEC_FROM_TEXT('[12.0]') < VEC_FROM_TEXT('[4.0]'); ++--------------------------------------------------+ +| VEC_FROM_TEXT('[12.0]') < VEC_FROM_TEXT('[4.0]') | ++--------------------------------------------------+ +| 0 | ++--------------------------------------------------+ +1 row in set (0.01 sec) +``` + +## Arithmetic + +Vector data types support arithmetic operations `+` (addition) and `-` (subtraction). However, arithmetic operations between vectors with different dimensions are not supported and will result in an error. + +Examples: + +```sql +[tidb]> SELECT VEC_FROM_TEXT('[4]') + VEC_FROM_TEXT('[5]'); ++---------------------------------------------+ +| VEC_FROM_TEXT('[4]') + VEC_FROM_TEXT('[5]') | ++---------------------------------------------+ +| [9] | ++---------------------------------------------+ +1 row in set (0.01 sec) + +[tidb]> SELECT VEC_FROM_TEXT('[2,3,4]') - VEC_FROM_TEXT('[1,2,3]'); ++-----------------------------------------------------+ +| VEC_FROM_TEXT('[2,3,4]') - VEC_FROM_TEXT('[1,2,3]') | ++-----------------------------------------------------+ +| [1,1,1] | ++-----------------------------------------------------+ +1 row in set (0.01 sec) + +[tidb]> SELECT VEC_FROM_TEXT('[4]') + VEC_FROM_TEXT('[1,2,3]'); +ERROR 1105 (HY000): vectors have different dimensions: 1 and 3 +``` + +## Cast + +### Cast between Vector ⇔ String + +To cast between Vector and String, use the following functions: + +- `CAST(... AS VECTOR)`: String ⇒ Vector +- `CAST(... AS CHAR)`: Vector ⇒ String +- `VEC_FROM_TEXT`: String ⇒ Vector +- `VEC_AS_TEXT`: Vector ⇒ String + +To improve usability, if you call a function that only supports vector data types, such as a vector correlation distance function, you can also just pass in a format-compliant string. TiDB automatically performs an implicit cast in this case. + +```sql +-- The VEC_DIMS function only accepts VECTOR arguments, so you can directly pass in a string for an implicit cast. +[tidb]> SELECT VEC_DIMS('[0.3, 0.5, -0.1]'); ++------------------------------+ +| VEC_DIMS('[0.3, 0.5, -0.1]') | ++------------------------------+ +| 3 | ++------------------------------+ +1 row in set (0.01 sec) + +-- You can also explicitly cast a string to a vector using VEC_FROM_TEXT and then pass the vector to the VEC_DIMS function. +[tidb]> SELECT VEC_DIMS(VEC_FROM_TEXT('[0.3, 0.5, -0.1]')); ++---------------------------------------------+ +| VEC_DIMS(VEC_FROM_TEXT('[0.3, 0.5, -0.1]')) | ++---------------------------------------------+ +| 3 | ++---------------------------------------------+ +1 row in set (0.01 sec) + +-- You can also cast explicitly using CAST(... AS VECTOR): +[tidb]> SELECT VEC_DIMS(CAST('[0.3, 0.5, -0.1]' AS VECTOR)); ++----------------------------------------------+ +| VEC_DIMS(CAST('[0.3, 0.5, -0.1]' AS VECTOR)) | ++----------------------------------------------+ +| 3 | ++----------------------------------------------+ +1 row in set (0.01 sec) +``` + +When using an operator or function that accepts multiple data types, you need to explicitly cast the string type to the vector type before passing the string to that operator or function, because TiDB does not perform implicit casts in this case. For example, before performing comparison operations, you need to explicitly cast strings to vectors; otherwise, TiDB compares them as string values rather than as vector numeric values: + +```sql +-- Because string is given, TiDB is comparing strings: +[tidb]> SELECT '[12.0]' < '[4.0]'; ++--------------------+ +| '[12.0]' < '[4.0]' | ++--------------------+ +| 1 | ++--------------------+ +1 row in set (0.01 sec) + +-- Cast to vector explicitly to compare by vectors: +[tidb]> SELECT VEC_FROM_TEXT('[12.0]') < VEC_FROM_TEXT('[4.0]'); ++--------------------------------------------------+ +| VEC_FROM_TEXT('[12.0]') < VEC_FROM_TEXT('[4.0]') | ++--------------------------------------------------+ +| 0 | ++--------------------------------------------------+ +1 row in set (0.01 sec) +``` + +You can also explicitly cast a vector to its string representation. Take using the `VEC_AS_TEXT()` function as an example: + +```sql +-- The string is first implicitly cast to a vector, and then the vector is explicitly cast to a string, thus returning a string in the normalized format: +[tidb]> SELECT VEC_AS_TEXT('[0.3, 0.5, -0.1]'); ++--------------------------------------+ +| VEC_AS_TEXT('[0.3, 0.5, -0.1]') | ++--------------------------------------+ +| [0.3,0.5,-0.1] | ++--------------------------------------+ +1 row in set (0.01 sec) +``` + +For additional cast functions, see [Vector Functions and Operators](/tidb-cloud/vector-search-functions-and-operators.md). + +### Cast between Vector ⇔ other data types + +Currently, direct casting between Vector and other data types (such as `JSON`) is not supported. To work around this limitation, use String as an intermediate data type for casting in your SQL statement. + +Note that vector data type columns stored in a table cannot be converted to other data types using `ALTER TABLE ... MODIFY COLUMN ...`. + +## Limitations + +See [Vector data type limitations](/tidb-cloud/vector-search-limitations.md#vector-data-type-limitations). + +## MySQL compatibility + +Vector data types are TiDB specific, and are not supported in MySQL. + +## See also + +- [Vector Functions and Operators](/tidb-cloud/vector-search-functions-and-operators.md) +- [Vector Search Index](/tidb-cloud/vector-search-index.md) +- [Improve Vector Search Performance](/tidb-cloud/vector-search-improve-performance.md) diff --git a/tidb-cloud/vector-search-functions-and-operators.md b/tidb-cloud/vector-search-functions-and-operators.md new file mode 100644 index 0000000000000..4081a2f4fcc45 --- /dev/null +++ b/tidb-cloud/vector-search-functions-and-operators.md @@ -0,0 +1,284 @@ +--- +title: Vector Functions and Operators +summary: Learn about functions and operators available for Vector data types. +--- + +# Vector Functions and Operators + +This document lists the functions and operators available for Vector data types. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## Vector functions + +The following functions are designed specifically for [Vector data types](/tidb-cloud/vector-search-data-types.md). + +**Vector distance functions:** + +| Function Name | Description | +| ----------------------------------------------------------- | ---------------------------------------------------------------- | +| [`VEC_L2_DISTANCE`](#vec_l2_distance) | Calculates L2 distance (Euclidean distance) between two vectors | +| [`VEC_COSINE_DISTANCE`](#vec_cosine_distance) | Calculates the cosine distance between two vectors | +| [`VEC_NEGATIVE_INNER_PRODUCT`](#vec_negative_inner_product) | Calculates the negative of the inner product between two vectors | +| [`VEC_L1_DISTANCE`](#vec_l1_distance) | Calculates L1 distance (Manhattan distance) between two vectors | + +**Other vector functions:** + +| Function Name | Description | +| --------------------------------- | --------------------------------------------------- | +| [`VEC_DIMS`](#vec_dims) | Returns the dimension of a vector | +| [`VEC_L2_NORM`](#vec_l2_norm) | Calculates the L2 norm (Euclidean norm) of a vector | +| [`VEC_FROM_TEXT`](#vec_from_text) | Converts a string into a vector | +| [`VEC_AS_TEXT`](#vec_as_text) | Converts a vector into a string | + +## Extended built-in functions and operators + +The following built-in functions and operators are extended to support operations on [Vector data types](/tidb-cloud/vector-search-data-types.md). + +**Arithmetic operators:** + +| Name | Description | +| :-------------------------------------------------------------------------------------- | :--------------------------------------- | +| [`+`](https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html#operator_plus) | Vector element-wise addition operator | +| [`-`](https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html#operator_minus) | Vector element-wise subtraction operator | + +For more information about how vector arithmetic works, see [Vector Data Type | Arithmetic](/tidb-cloud/vector-search-data-types.md#arithmetic). + +**Aggregate (GROUP BY) functions:** + +| Name | Description | +| :------------------------------------------------------------------------------------------------------------ | :----------------------------------------------- | +| [`COUNT()`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_count) | Return a count of the number of rows returned | +| [`COUNT(DISTINCT)`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_count-distinct) | Return the count of a number of different values | +| [`MAX()`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_max) | Return the maximum value | +| [`MIN()`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_min) | Return the minimum value | + +**Comparison functions and operators:** + +| Name | Description | +| ------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------- | +| [`BETWEEN ... AND ...`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_between) | Check whether a value is within a range of values | +| [`COALESCE()`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#function_coalesce) | Return the first non-NULL argument | +| [`=`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_equal) | Equal operator | +| [`<=>`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_equal-to) | NULL-safe equal to operator | +| [`>`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_greater-than) | Greater than operator | +| [`>=`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_greater-than-or-equal) | Greater than or equal operator | +| [`GREATEST()`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#function_greatest) | Return the largest argument | +| [`IN()`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_in) | Check whether a value is within a set of values | +| [`IS NULL`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_is-null) | Test whether a value is `NULL` | +| [`ISNULL()`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#function_isnull) | Test whether the argument is `NULL` | +| [`LEAST()`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#function_least) | Return the smallest argument | +| [`<`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_less-than) | Less than operator | +| [`<=`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_less-than-or-equal) | Less than or equal operator | +| [`NOT BETWEEN ... AND ...`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_not-between) | Check whether a value is not within a range of values | +| [`!=`, `<>`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_not-equal) | Not equal operator | +| [`NOT IN()`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_not-in) | Check whether a value is not within a set of values | + +For more information about how vectors are compared, see [Vector Data Type | Comparison](/tidb-cloud/vector-search-data-types.md#comparison). + +**Control flow functions:** + +| Name | Description | +| :------------------------------------------------------------------------------------------------ | :----------------------------- | +| [`CASE`](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#operator_case) | Case operator | +| [`IF()`](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#function_if) | If/else construct | +| [`IFNULL()`](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#function_ifnull) | Null if/else construct | +| [`NULLIF()`](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#function_nullif) | Return `NULL` if expr1 = expr2 | + +**Cast functions:** + +| Name | Description | +| :------------------------------------------------------------------------------------------ | :--------------------------------- | +| [`CAST()`](https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html#function_cast) | Cast a value as a string or vector | +| [`CONVERT()`](https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html#function_convert) | Cast a value as a string | + +For more information about how to use `CAST()`, see [Vector Data Type | Cast](/tidb-cloud/vector-search-data-types.md#cast). + +## Full references + +### VEC_L2_DISTANCE + +```sql +VEC_L2_DISTANCE(vector1, vector2) +``` + +Calculates the [L2 distance](https://en.wikipedia.org/wiki/Euclidean_distance) (Euclidean distance) between two vectors using the following formula: + +$DISTANCE(p,q)=\sqrt {\sum \limits _{i=1}^{n}{(p_{i}-q_{i})^{2}}}$ + +The two vectors must have the same dimension. Otherwise, an error is returned. + +Example: + +```sql +[tidb]> SELECT VEC_L2_DISTANCE('[0,3]', '[4,0]'); ++-----------------------------------+ +| VEC_L2_DISTANCE('[0,3]', '[4,0]') | ++-----------------------------------+ +| 5 | ++-----------------------------------+ +``` + +### VEC_COSINE_DISTANCE + +```sql +VEC_COSINE_DISTANCE(vector1, vector2) +``` + +Calculates the [cosine distance](https://en.wikipedia.org/wiki/Cosine_similarity) between two vectors using the following formula: + +$DISTANCE(p,q)=1.0 - {\frac {\sum \limits _{i=1}^{n}{p_{i}q_{i}}}{{\sqrt {\sum \limits _{i=1}^{n}{p_{i}^{2}}}}\cdot {\sqrt {\sum \limits _{i=1}^{n}{q_{i}^{2}}}}}}$ + +The two vectors must have the same dimension. Otherwise, an error is returned. + +Example: + +```sql +[tidb]> SELECT VEC_COSINE_DISTANCE('[1, 1]', '[-1, -1]'); ++-------------------------------------------+ +| VEC_COSINE_DISTANCE('[1, 1]', '[-1, -1]') | ++-------------------------------------------+ +| 2 | ++-------------------------------------------+ +``` + +### VEC_NEGATIVE_INNER_PRODUCT + +```sql +VEC_NEGATIVE_INNER_PRODUCT(vector1, vector2) +``` + +Calculates the distance by using the negative of the [inner product](https://en.wikipedia.org/wiki/Dot_product) between two vectors, using the following formula: + +$DISTANCE(p,q)=- INNER\_PROD(p,q)=-\sum \limits _{i=1}^{n}{p_{i}q_{i}}$ + +The two vectors must have the same dimension. Otherwise, an error is returned. + +Example: + +```sql +[tidb]> SELECT VEC_NEGATIVE_INNER_PRODUCT('[1,2]', '[3,4]'); ++----------------------------------------------+ +| VEC_NEGATIVE_INNER_PRODUCT('[1,2]', '[3,4]') | ++----------------------------------------------+ +| -11 | ++----------------------------------------------+ +``` + +### VEC_L1_DISTANCE + +```sql +VEC_L1_DISTANCE(vector1, vector2) +``` + +Calculates the [L1 distance](https://en.wikipedia.org/wiki/Taxicab_geometry) (Manhattan distance) between two vectors using the following formula: + +$DISTANCE(p,q)=\sum \limits _{i=1}^{n}{|p_{i}-q_{i}|}$ + +The two vectors must have the same dimension. Otherwise, an error is returned. + +Example: + +```sql +[tidb]> SELECT VEC_L1_DISTANCE('[0,0]', '[3,4]'); ++-----------------------------------+ +| VEC_L1_DISTANCE('[0,0]', '[3,4]') | ++-----------------------------------+ +| 7 | ++-----------------------------------+ +``` + +### VEC_DIMS + +```sql +VEC_DIMS(vector) +``` + +Returns the dimension of a vector. + +Examples: + +```sql +[tidb]> SELECT VEC_DIMS('[1,2,3]'); ++---------------------+ +| VEC_DIMS('[1,2,3]') | ++---------------------+ +| 3 | ++---------------------+ + +[tidb]> SELECT VEC_DIMS('[]'); ++----------------+ +| VEC_DIMS('[]') | ++----------------+ +| 0 | ++----------------+ +``` + +### VEC_L2_NORM + +```sql +VEC_L2_NORM(vector) +``` + +Calculates the [L2 norm]() (Euclidean norm) of a vector using the following formula: + +$NORM(p)=\sqrt {\sum \limits _{i=1}^{n}{p_{i}^{2}}}$ + +Example: + +```sql +[tidb]> SELECT VEC_L2_NORM('[3,4]'); ++----------------------+ +| VEC_L2_NORM('[3,4]') | ++----------------------+ +| 5 | ++----------------------+ +``` + +### VEC_FROM_TEXT + +```sql +VEC_FROM_TEXT(string) +``` + +Converts a string into a vector. + +Example: + +```sql +[tidb]> SELECT VEC_FROM_TEXT('[1,2]') + VEC_FROM_TEXT('[3,4]'); ++-------------------------------------------------+ +| VEC_FROM_TEXT('[1,2]') + VEC_FROM_TEXT('[3,4]') | ++-------------------------------------------------+ +| [4,6] | ++-------------------------------------------------+ +``` + +### VEC_AS_TEXT + +```sql +VEC_AS_TEXT(vector) +``` + +Converts a vector into a string. + +Example: + +```sql +[tidb]> SELECT VEC_AS_TEXT('[1.000, 2.5]'); ++-------------------------------+ +| VEC_AS_TEXT('[1.000, 2.5]') | ++-------------------------------+ +| [1,2.5] | ++-------------------------------+ +``` + +## MySQL compatibility + +The vector functions and the extended usage of built-in functions and operators over vector data types are TiDB specific, and are not supported in MySQL. + +## See also + +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) diff --git a/tidb-cloud/vector-search-get-started-using-python.md b/tidb-cloud/vector-search-get-started-using-python.md new file mode 100644 index 0000000000000..be7f8e0ff236d --- /dev/null +++ b/tidb-cloud/vector-search-get-started-using-python.md @@ -0,0 +1,195 @@ +--- +title: Get Started with TiDB + AI via Python +summary: Learn how to quickly develop an AI application that performs semantic search using Python and TiDB Vector Search. +--- + +# Get Started with TiDB + AI via Python + +This tutorial demonstrates how to develop a simple AI application that provides **semantic search** features. Unlike traditional keyword search, semantic search intelligently understands the meaning behind your query and returns the most relevant result. For example, if you have documents titled "dog", "fish", and "tree", and you search for "a swimming animal", the application would identify "fish" as the most relevant result. + +Throughout this tutorial, you will develop this AI application using [TiDB Vector Search](/tidb-cloud/vector-search-overview.md), Python, [TiDB Vector SDK for Python](https://github.com/pingcap/tidb-vector-python), and AI models. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## Prerequisites + +To complete this tutorial, you need: + +- [Python 3.8 or higher](https://www.python.org/downloads/) installed. +- [Git](https://git-scm.com/downloads) installed. +- A TiDB Cloud Serverless cluster. Follow [creating a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create your own TiDB Cloud cluster if you don't have one. + +## Get started + +The following steps show how to develop the application from scratch. To run the demo directly, you can check out the sample code in the [pingcap/tidb-vector-python](https://github.com/pingcap/tidb-vector-python/blob/main/examples/python-client-quickstart) repository. + +### Step 1. Create a new Python project + +In your preferred directory, create a new Python project and a file named `example.py`: + +```shell +mkdir python-client-quickstart +cd python-client-quickstart +touch example.py +``` + +### Step 2. Install required dependencies + +In your project directory, run the following command to install the required packages: + +```shell +pip install sqlalchemy pymysql sentence-transformers tidb-vector python-dotenv +``` + +- `tidb-vector`: the Python client for interacting with TiDB Vector Search. +- [`sentence-transformers`](https://sbert.net): a Python library that provides pre-trained models for generating [vector embeddings](/tidb-cloud/vector-search-overview.md#vector-embedding) from text. + +### Step 3. Configure the connection string to the TiDB cluster + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Ensure the configurations in the connection dialog match your operating environment. + + - **Connection Type** is set to `Public`. + - **Branch** is set to `main`. + - **Connect With** is set to `SQLAlchemy`. + - **Operating System** matches your environment. + + > **Tip:** + > + > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. + +4. Click the **PyMySQL** tab and copy the connection string. + + > **Tip:** + > + > If you have not set a password yet, click **Generate Password** to generate a random password. + +5. In the root directory of your Python project, create a `.env` file and paste the connection string into it. + + The following is an example for macOS: + + ```dotenv + TIDB_DATABASE_URL="mysql+pymysql://.root:@gateway01..prod.aws.tidbcloud.com:4000/test?ssl_ca=/etc/ssl/cert.pem&ssl_verify_cert=true&ssl_verify_identity=true" + ``` + +### Step 4. Initialize the embedding model + +An [embedding model](/tidb-cloud/vector-search-overview.md#embedding-model) transforms data into [vector embeddings](/tidb-cloud/vector-search-overview.md#vector-embedding). This example uses the pre-trained model [**msmarco-MiniLM-L12-cos-v5**](https://huggingface.co/sentence-transformers/msmarco-MiniLM-L12-cos-v5) for text embedding. This lightweight model, provided by the `sentence-transformers` library, transforms text data into 384-dimensional vector embeddings. + +To set up the model, copy the following code into the `example.py` file. This code initializes a `SentenceTransformer` instance and defines a `text_to_embedding()` function for later use. + +```python +from sentence_transformers import SentenceTransformer + +print("Downloading and loading the embedding model...") +embed_model = SentenceTransformer("sentence-transformers/msmarco-MiniLM-L12-cos-v5", trust_remote_code=True) +embed_model_dims = embed_model.get_sentence_embedding_dimension() + +def text_to_embedding(text): + """Generates vector embeddings for the given text.""" + embedding = embed_model.encode(text) + return embedding.tolist() +``` + +### Step 5. Connect to the TiDB cluster + +Use the `TiDBVectorClient` class to connect to your TiDB cluster and create a table `embedded_documents` with a vector column. + +> **Note** +> +> Make sure the dimension of your vector column in the table matches the dimension of the vectors generated by your embedding model. For example, the **msmarco-MiniLM-L12-cos-v5** model generates vectors with 384 dimensions, so the dimension of your vector columns in `embedded_documents` should be 384 as well. + +```python +import os +from tidb_vector.integrations import TiDBVectorClient +from dotenv import load_dotenv + +# Load the connection string from the .env file +load_dotenv() + +vector_store = TiDBVectorClient( + # The 'embedded_documents' table will store the vector data. + table_name='embedded_documents', + # The connection string to the TiDB cluster. + connection_string=os.environ.get('TIDB_DATABASE_URL'), + # The dimension of the vector generated by the embedding model. + vector_dimension=embed_model_dims, + # Recreate the table if it already exists. + drop_existing_table=True, +) +``` + +### Step 6. Embed text data and store the vectors + +In this step, you will prepare sample documents containing single words, such as "dog", "fish", and "tree". The following code uses the `text_to_embedding()` function to transform these text documents into vector embeddings, and then inserts them into the vector store. + +```python +documents = [ + { + "id": "f8e7dee2-63b6-42f1-8b60-2d46710c1971", + "text": "dog", + "embedding": text_to_embedding("dog"), + "metadata": {"category": "animal"}, + }, + { + "id": "8dde1fbc-2522-4ca2-aedf-5dcb2966d1c6", + "text": "fish", + "embedding": text_to_embedding("fish"), + "metadata": {"category": "animal"}, + }, + { + "id": "e4991349-d00b-485c-a481-f61695f2b5ae", + "text": "tree", + "embedding": text_to_embedding("tree"), + "metadata": {"category": "plant"}, + }, +] + +vector_store.insert( + ids=[doc["id"] for doc in documents], + texts=[doc["text"] for doc in documents], + embeddings=[doc["embedding"] for doc in documents], + metadatas=[doc["metadata"] for doc in documents], +) +``` + +### Step 7. Perform semantic search + +In this step, you will search for "a swimming animal", which doesn't directly match any words in existing documents. + +The following code uses the `text_to_embedding()` function again to convert the query text into a vector embedding, and then queries with the embedding to find the top three closest matches. + +```python +def print_result(query, result): + print(f"Search result (\"{query}\"):") + for r in result: + print(f"- text: \"{r.document}\", distance: {r.distance}") + +query = "a swimming animal" +query_embedding = text_to_embedding(query) +search_result = vector_store.query(query_embedding, k=3) +print_result(query, search_result) +``` + +Run the `example.py` file and the output is as follows: + +```plain +Search result ("a swimming animal"): +- text: "fish", distance: 0.4562914811223072 +- text: "dog", distance: 0.6469335836410557 +- text: "tree", distance: 0.798545178640937 +``` + +The three terms in the search results are sorted by their respective distance from the queried vector: the smaller the distance, the more relevant the corresponding `document`. + +Therefore, according to the output, the swimming animal is most likely a fish, or a dog with a gift for swimming. + +## See also + +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) +- [Vector Search Index](/tidb-cloud/vector-search-index.md) diff --git a/tidb-cloud/vector-search-get-started-using-sql.md b/tidb-cloud/vector-search-get-started-using-sql.md new file mode 100644 index 0000000000000..7546b060a5c79 --- /dev/null +++ b/tidb-cloud/vector-search-get-started-using-sql.md @@ -0,0 +1,150 @@ +--- +title: Get Started with Vector Search via SQL +summary: Learn how to quickly get started with Vector Search in TiDB using SQL statements to power your generative AI applications. +--- + +# Get Started with Vector Search via SQL + +TiDB extends MySQL syntax to support [Vector Search](/tidb-cloud/vector-search-overview.md) and introduce new [Vector data types](/tidb-cloud/vector-search-data-types.md) and several [vector functions](/tidb-cloud/vector-search-functions-and-operators.md). + +This tutorial demonstrates how to get started with TiDB Vector Search just using SQL statements. You will learn how to use the [MySQL command-line client](https://dev.mysql.com/doc/refman/8.4/en/mysql.html) to complete the following operations: + +- Connect to your TiDB cluster. +- Create a vector table. +- Store vector embeddings. +- Perform vector search queries. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## Prerequisites + +To complete this tutorial, you need: + +- [MySQL command-line client](https://dev.mysql.com/doc/refman/8.4/en/mysql.html) (MySQL CLI) installed on your machine. +- A TiDB Cloud Serverless cluster. Follow [creating a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create your own TiDB Cloud cluster if you don't have one. + +## Get started + +### Step 1. Connect to the TiDB cluster + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. In the connection dialog, select **MySQL CLI** from the **Connect With** drop-down list and keep the default setting of the **Connection Type** as **Public**. + +4. If you have not set a password yet, click **Generate Password** to generate a random password. + +5. Copy the connection command and paste it into your terminal. The following is an example for macOS: + + ```bash + mysql -u '.root' -h '' -P 4000 -D 'test' --ssl-mode=VERIFY_IDENTITY --ssl-ca=/etc/ssl/cert.pem -p'' + ``` + +### Step 2. Create a vector table + +When creating a table, you can define a column as a [vector](/tidb-cloud/vector-search-overview.md#vector-embedding) column by specifying the `VECTOR` data type. + +For example, to create a table `embedded_documents` with a three-dimensional `VECTOR` column, execute the following SQL statements using your MySQL CLI: + +```sql +USE test; +CREATE TABLE embedded_documents ( + id INT PRIMARY KEY, + -- Column to store the original content of the document. + document TEXT, + -- Column to store the vector representation of the document. + embedding VECTOR(3) +); +``` + +The expected output is as follows: + +```text +Query OK, 0 rows affected (0.27 sec) +``` + +### Step 3. Insert vector embeddings to the table + +Insert three documents with their [vector embeddings](/tidb-cloud/vector-search-overview.md#vector-embedding) into the `embedded_documents` table: + +```sql +INSERT INTO embedded_documents +VALUES + (1, 'dog', '[1,2,1]'), + (2, 'fish', '[1,2,4]'), + (3, 'tree', '[1,0,0]'); +``` + +The expected output is as follows: + +``` +Query OK, 3 rows affected (0.15 sec) +Records: 3 Duplicates: 0 Warnings: 0 +``` + +> **Note** +> +> This example simplifies the dimensions of the vector embeddings and uses only 3-dimensional vectors for demonstration purposes. +> +> In real-world applications, [embedding models](/tidb-cloud/vector-search-overview.md#embedding-model) often produce vector embeddings with hundreds or thousands of dimensions. + +### Step 4. Query the vector table + +To verify that the documents have been inserted correctly, query the `embedded_documents` table: + +```sql +SELECT * FROM embedded_documents; +``` + +The expected output is as follows: + +```sql ++----+----------+-----------+ +| id | document | embedding | ++----+----------+-----------+ +| 1 | dog | [1,2,1] | +| 2 | fish | [1,2,4] | +| 3 | tree | [1,0,0] | ++----+----------+-----------+ +3 rows in set (0.15 sec) +``` + +### Step 5. Perform a vector search query + +Similar to full-text search, users provide search terms to the application when using vector search. + +In this example, the search term is "a swimming animal", and its corresponding vector embedding is assumed to be `[1,2,3]`. In practical applications, you need to use an embedding model to convert the user's search term into a vector embedding. + +Execute the following SQL statement, and TiDB will identify the top three documents closest to `[1,2,3]` by calculating and sorting the cosine distances (`vec_cosine_distance`) between the vector embeddings in the table. + +```sql +SELECT id, document, vec_cosine_distance(embedding, '[1,2,3]') AS distance +FROM embedded_documents +ORDER BY distance +LIMIT 3; +``` + +The expected output is as follows: + +```plain ++----+----------+---------------------+ +| id | document | distance | ++----+----------+---------------------+ +| 2 | fish | 0.00853986601633272 | +| 1 | dog | 0.12712843905603044 | +| 3 | tree | 0.7327387580875756 | ++----+----------+---------------------+ +3 rows in set (0.15 sec) +``` + +The three terms in the search results are sorted by their respective distance from the queried vector: the smaller the distance, the more relevant the corresponding `document`. + +Therefore, according to the output, the swimming animal is most likely a fish, or a dog with a gift for swimming. + +## See also + +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) +- [Vector Search Index](/tidb-cloud/vector-search-index.md) diff --git a/tidb-cloud/vector-search-improve-performance.md b/tidb-cloud/vector-search-improve-performance.md new file mode 100644 index 0000000000000..70fd40f468e6a --- /dev/null +++ b/tidb-cloud/vector-search-improve-performance.md @@ -0,0 +1,36 @@ +--- +title: Improve Vector Search Performance +summary: Learn best practices for improving the performance of TiDB Vector Search. +--- + +# Improve Vector Search Performance + +TiDB Vector Search enables you to perform Approximate Nearest Neighbor (ANN) queries that search for results similar to an image, document, or other input. To improve the query performance, review the following best practices. + +## Add vector search index for vector columns + +The [vector search index](/tidb-cloud/vector-search-index.md) dramatically improves the performance of vector search queries, usually by 10x or more, with a trade-off of only a small decrease of recall rate. + +## Ensure vector indexes are fully built + +After you insert a large volume of vector data, some of it might be in the Delta layer waiting for persistence. The vector index for such data will be built after the data is persisted. Until all vector data is indexed, vector search performance is suboptimal. To check the index build progress, see [View index build progress](/tidb-cloud/vector-search-index.md#view-index-build-progress). + +## Reduce vector dimensions or shorten embeddings + +The computational complexity of vector search indexing and queries increases significantly as the dimension of vectors grows, requiring more floating-point comparisons. + +To optimize performance, consider reducing vector dimensions whenever feasible. This usually needs switching to another embedding model. When switching models, you need to evaluate the impact of the model change on the accuracy of vector queries. + +Certain embedding models like OpenAI `text-embedding-3-large` support [shortening embeddings](https://openai.com/index/new-embedding-models-and-api-updates/), which removes some numbers from the end of vector sequences without losing the embedding's concept-representing properties. You can also use such an embedding model to reduce the vector dimensions. + +## Exclude vector columns from the results + +Vector embedding data is usually large and only used during the search process. By excluding vector columns from query results, you can greatly reduce the data transferred between the TiDB server and your SQL client, thereby improving query performance. + +To exclude vector columns, explicitly list the columns you want to retrieve in the `SELECT` clause, instead of using `SELECT *` to retrieve all columns. + +## Warm up the index + +When accessing an index that has never been used or has not been accessed for a long time (cold access), TiDB needs to load the entire index from cloud storage or disk (instead of from memory). This process takes time and often results in higher query latency. Additionally, if there are no SQL queries for an extended period (for example, several hours), computing resources are reclaimed, causing subsequent access to become cold access. + +To avoid such query latency, warm up your index before actual workload by running similar vector search queries that hit the vector index. \ No newline at end of file diff --git a/tidb-cloud/vector-search-index.md b/tidb-cloud/vector-search-index.md new file mode 100644 index 0000000000000..2da5b3d5a63fd --- /dev/null +++ b/tidb-cloud/vector-search-index.md @@ -0,0 +1,245 @@ +--- +title: Vector Search Index +summary: Learn how to build and use the vector search index to accelerate K-Nearest neighbors (KNN) queries in TiDB. +--- + +# Vector Search Index + +As described in the [Vector Search](/tidb-cloud/vector-search-overview.md) document, vector search identifies the Top K-Nearest Neighbors (KNN) to a given vector by calculating the distance between the given vector and all vectors stored in the database. While this approach provides accurate results, it can be slow when the table contains a large number of vectors because it involves a full table scan. [^1] + +To improve search efficiency, you can create vector search indexes in TiDB for approximate KNN (ANN) search. When using vector indexes for vector search, TiDB can greatly improve query performance with only a slight reduction in accuracy, generally maintaining a search recall rate above 90%. + +Currently, TiDB supports the [HNSW (Hierarchical Navigable Small World)](https://en.wikipedia.org/wiki/Hierarchical_navigable_small_world) vector search index algorithm. + +## Create the HNSW vector index + +[HNSW](https://en.wikipedia.org/wiki/Hierarchical_navigable_small_world) is one of the most popular vector indexing algorithms. The HNSW index provides good performance with relatively high accuracy, up to 98% in specific cases. + +In TiDB, you can create an HNSW index for a column with a [vector data type](/tidb-cloud/vector-search-data-types.md) in either of the following ways: + +- When creating a table, use the following syntax to specify the vector column for the HNSW index: + + ```sql + CREATE TABLE foo ( + id INT PRIMARY KEY, + embedding VECTOR(5), + VECTOR INDEX idx_embedding ((VEC_COSINE_DISTANCE(embedding))) + ); + ``` + +- For an existing table that already contains a vector column, use the following syntax to create an HNSW index for the vector column: + + ```sql + CREATE VECTOR INDEX idx_embedding ON foo ((VEC_COSINE_DISTANCE(embedding))); + ALTER TABLE foo ADD VECTOR INDEX idx_embedding ((VEC_COSINE_DISTANCE(embedding))); + + -- You can also explicitly specify "USING HNSW" to build the vector search index. + CREATE VECTOR INDEX idx_embedding ON foo ((VEC_COSINE_DISTANCE(embedding))) USING HNSW; + ALTER TABLE foo ADD VECTOR INDEX idx_embedding ((VEC_COSINE_DISTANCE(embedding))) USING HNSW; + ``` + +> **Note:** +> +> The vector search index feature relies on TiFlash replicas for tables. +> +> - If a vector search index is defined when a table is created, TiDB automatically creates a TiFlash replica for the table. +> - If no vector search index is defined when a table is created, and the table currently does not have a TiFlash replica, you need to manually create a TiFlash replica before adding a vector search index to the table. For example: `ALTER TABLE 'table_name' SET TIFLASH REPLICA 1;`. + +When creating an HNSW vector index, you need to specify the distance function for the vector: + +- Cosine Distance: `((VEC_COSINE_DISTANCE(embedding)))` +- L2 Distance: `((VEC_L2_DISTANCE(embedding)))` + +The vector index can only be created for fixed-dimensional vector columns, such as a column defined as `VECTOR(3)`. It cannot be created for non-fixed-dimensional vector columns (such as a column defined as `VECTOR`) because vector distances can only be calculated between vectors with the same dimension. + +For other limitations, see [Vector index limitations](/tidb-cloud/vector-search-limitations.md#vector-index-limitations). + +## Use the vector index + +The vector search index can be used in K-nearest neighbor search queries by using the `ORDER BY ... LIMIT` clause as follows: + +```sql +SELECT * +FROM foo +ORDER BY VEC_COSINE_DISTANCE(embedding, '[1, 2, 3, 4, 5]') +LIMIT 10 +``` + +To use an index in a vector search, make sure that the `ORDER BY ... LIMIT` clause uses the same distance function as the one specified when creating the vector index. + +## Use the vector index with filters + +Queries that contain a pre-filter (using the `WHERE` clause) cannot utilize the vector index because they are not querying for K-Nearest neighbors according to the SQL semantics. For example: + +```sql +-- For the following query, the `WHERE` filter is performed before KNN, so the vector index cannot be used: + +SELECT * FROM vec_table +WHERE category = "document" +ORDER BY VEC_COSINE_DISTANCE(embedding, '[1, 2, 3]') +LIMIT 5; +``` + +To use the vector index with filters, query for the K-Nearest neighbors first using vector search, and then filter out unwanted results: + +```sql +-- For the following query, the `WHERE` filter is performed after KNN, so the vector index cannot be used: + +SELECT * FROM +( + SELECT * FROM vec_table + ORDER BY VEC_COSINE_DISTANCE(embedding, '[1, 2, 3]') + LIMIT 5 +) t +WHERE category = "document"; + +-- Note that this query might return fewer than 5 results if some are filtered out. +``` + +## View index build progress + +After you insert a large volume of data, some of it might not be instantly persisted to TiFlash. For vector data that has already been persisted, the vector search index is built synchronously. For data that has not yet been persisted, the index will be built once the data is persisted. This process does not affect the accuracy and consistency of the data. You can still perform vector searches at any time and get complete results. However, performance will be suboptimal until vector indexes are fully built. + +To view the index build progress, you can query the `INFORMATION_SCHEMA.TIFLASH_INDEXES` table as follows: + +```sql +SELECT * FROM INFORMATION_SCHEMA.TIFLASH_INDEXES; ++---------------+------------+----------+-------------+---------------+-----------+----------+------------+---------------------+-------------------------+--------------------+------------------------+---------------+------------------+ +| TIDB_DATABASE | TIDB_TABLE | TABLE_ID | COLUMN_NAME | INDEX_NAME | COLUMN_ID | INDEX_ID | INDEX_KIND | ROWS_STABLE_INDEXED | ROWS_STABLE_NOT_INDEXED | ROWS_DELTA_INDEXED | ROWS_DELTA_NOT_INDEXED | ERROR_MESSAGE | TIFLASH_INSTANCE | ++---------------+------------+----------+-------------+---------------+-----------+----------+------------+---------------------+-------------------------+--------------------+------------------------+---------------+------------------+ +| test | tcff1d827 | 219 | col1fff | 0a452311 | 7 | 1 | HNSW | 29646 | 0 | 0 | 0 | | 127.0.0.1:3930 | +| test | foo | 717 | embedding | idx_embedding | 2 | 1 | HNSW | 0 | 0 | 0 | 3 | | 127.0.0.1:3930 | ++---------------+------------+----------+-------------+---------------+-----------+----------+------------+---------------------+-------------------------+--------------------+------------------------+---------------+------------------+ +``` + +- You can check the `ROWS_STABLE_INDEXED` and `ROWS_STABLE_NOT_INDEXED` columns for the index build progress. When `ROWS_STABLE_NOT_INDEXED` becomes 0, the index build is complete. + + As a reference, indexing a 500 MiB vector dataset with 768 dimensions might take up to 20 minutes. The indexer can run in parallel for multiple tables. Currently, adjusting the indexer priority or speed is not supported. + +- You can check the `ROWS_DELTA_NOT_INDEXED` column for the number of rows in the Delta layer. Data in the storage layer of TiFlash is stored in two layers: Delta layer and Stable layer. The Delta layer stores recently inserted or updated rows and is periodically merged into the Stable layer according to the write workload. This merge process is called Compaction. + + The Delta layer is always not indexed. To achieve optimal performance, you can force the merge of the Delta layer into the Stable layer so that all data can be indexed: + + ```sql + ALTER TABLE COMPACT; + ``` + + For more information, see [`ALTER TABLE ... COMPACT`](/sql-statements/sql-statement-alter-table-compact.md). + +In addition, you can monitor the execution progress of the DDL job by executing `ADMIN SHOW DDL JOBS;` and checking the `row count`. However, this method is not fully accurate, because the `row count` value is obtained from the `rows_stable_indexed` field in `TIFLASH_INDEXES`. You can use this approach as a reference for tracking the progress of indexing. + +## Check whether the vector index is used + +Use the [`EXPLAIN`](/sql-statements/sql-statement-explain.md) or [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) statement to check whether a query is using the vector index. When `annIndex:` is presented in the `operator info` column for the `TableFullScan` executor, it means this table scan is utilizing the vector index. + +**Example: the vector index is used** + +```sql +[tidb]> EXPLAIN SELECT * FROM vector_table_with_index +ORDER BY VEC_COSINE_DISTANCE(embedding, '[1, 2, 3]') +LIMIT 10; ++-----+-------------------------------------------------------------------------------------+ +| ... | operator info | ++-----+-------------------------------------------------------------------------------------+ +| ... | ... | +| ... | Column#5, offset:0, count:10 | +| ... | ..., vec_cosine_distance(test.vector_table_with_index.embedding, [1,2,3])->Column#5 | +| ... | MppVersion: 1, data:ExchangeSender_16 | +| ... | ExchangeType: PassThrough | +| ... | ... | +| ... | Column#4, offset:0, count:10 | +| ... | ..., vec_cosine_distance(test.vector_table_with_index.embedding, [1,2,3])->Column#4 | +| ... | annIndex:COSINE(test.vector_table_with_index.embedding..[1,2,3], limit:10), ... | ++-----+-------------------------------------------------------------------------------------+ +9 rows in set (0.01 sec) +``` + +**Example: The vector index is not used because of not specifying a Top K** + +```sql +[tidb]> EXPLAIN SELECT * FROM vector_table_with_index + -> ORDER BY VEC_COSINE_DISTANCE(embedding, '[1, 2, 3]'); ++--------------------------------+-----+--------------------------------------------------+ +| id | ... | operator info | ++--------------------------------+-----+--------------------------------------------------+ +| Projection_15 | ... | ... | +| └─Sort_4 | ... | Column#4 | +| └─Projection_16 | ... | ..., vec_cosine_distance(..., [1,2,3])->Column#4 | +| └─TableReader_14 | ... | MppVersion: 1, data:ExchangeSender_13 | +| └─ExchangeSender_13 | ... | ExchangeType: PassThrough | +| └─TableFullScan_12 | ... | keep order:false, stats:pseudo | ++--------------------------------+-----+--------------------------------------------------+ +6 rows in set, 1 warning (0.01 sec) +``` + +When the vector index cannot be used, a warning occurs in some cases to help you learn the cause: + +```sql +-- Using a wrong distance function: +[tidb]> EXPLAIN SELECT * FROM vector_table_with_index +ORDER BY VEC_L2_DISTANCE(embedding, '[1, 2, 3]') +LIMIT 10; + +[tidb]> SHOW WARNINGS; +ANN index not used: not ordering by COSINE distance + +-- Using a wrong order: +[tidb]> EXPLAIN SELECT * FROM vector_table_with_index +ORDER BY VEC_COSINE_DISTANCE(embedding, '[1, 2, 3]') DESC +LIMIT 10; + +[tidb]> SHOW WARNINGS; +ANN index not used: index can be used only when ordering by vec_cosine_distance() in ASC order +``` + +## Analyze vector search performance + +To learn detailed information about how a vector index is used, you can execute the [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) statement and check the `execution info` column in the output: + +```sql +[tidb]> EXPLAIN ANALYZE SELECT * FROM vector_table_with_index +ORDER BY VEC_COSINE_DISTANCE(embedding, '[1, 2, 3]') +LIMIT 10; ++-----+--------------------------------------------------------+-----+ +| | execution info | | ++-----+--------------------------------------------------------+-----+ +| ... | time:339.1ms, loops:2, RU:0.000000, Concurrency:OFF | ... | +| ... | time:339ms, loops:2 | ... | +| ... | time:339ms, loops:3, Concurrency:OFF | ... | +| ... | time:339ms, loops:3, cop_task: {...} | ... | +| ... | tiflash_task:{time:327.5ms, loops:1, threads:4} | ... | +| ... | tiflash_task:{time:327.5ms, loops:1, threads:4} | ... | +| ... | tiflash_task:{time:327.5ms, loops:1, threads:4} | ... | +| ... | tiflash_task:{time:327.5ms, loops:1, threads:4} | ... | +| ... | tiflash_task:{...}, vector_idx:{ | ... | +| | load:{total:68ms,from_s3:1,from_disk:0,from_cache:0},| | +| | search:{total:0ms,visited_nodes:2,discarded_nodes:0},| | +| | read:{vec_total:0ms,others_total:0ms}},...} | | ++-----+--------------------------------------------------------+-----+ +``` + +> **Note:** +> +> The execution information is internal. Fields and formats are subject to change without any notification. Do not rely on them. + +Explanation of some important fields: + +- `vector_index.load.total`: The total duration of loading index. This field might be larger than the actual query time because multiple vector indexes might be loaded in parallel. +- `vector_index.load.from_s3`: Number of indexes loaded from S3. +- `vector_index.load.from_disk`: Number of indexes loaded from disk. The index was already downloaded from S3 previously. +- `vector_index.load.from_cache`: Number of indexes loaded from cache. The index was already downloaded from S3 previously. +- `vector_index.search.total`: The total duration of searching in the index. Large latency usually means the index is cold (never accessed before, or accessed long ago) so that there are heavy I/O operations when searching through the index. This field might be larger than the actual query time because multiple vector indexes might be searched in parallel. +- `vector_index.search.discarded_nodes`: Number of vector rows visited but discarded during the search. These discarded vectors are not considered in the search result. Large values usually indicate that there are many stale rows caused by `UPDATE` or `DELETE` statements. + +See [`EXPLAIN`](/sql-statements/sql-statement-explain.md), [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md), and [EXPLAIN Walkthrough](/explain-walkthrough.md) for interpreting the output. + +## Limitations + +See [Vector index limitations](/tidb-cloud/vector-search-limitations.md#vector-index-limitations). + +## See also + +- [Improve Vector Search Performance](/tidb-cloud/vector-search-improve-performance.md) +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) + +[^1]: The explanation of KNN search is adapted from the [Approximate Nearest Neighbor Search Indexes](https://github.com/ClickHouse/ClickHouse/pull/50661/files#diff-7ebd9e71df96e74230c9a7e604fa7cb443be69ba5e23bf733fcecd4cc51b7576) document authored by [rschu1ze](https://github.com/rschu1ze) in ClickHouse documentation, licensed under the Apache License 2.0. diff --git a/tidb-cloud/vector-search-integrate-with-django-orm.md b/tidb-cloud/vector-search-integrate-with-django-orm.md new file mode 100644 index 0000000000000..61c9012ca54d8 --- /dev/null +++ b/tidb-cloud/vector-search-integrate-with-django-orm.md @@ -0,0 +1,221 @@ +--- +title: Integrate TiDB Vector Search with Django ORM +summary: Learn how to integrate TiDB Vector Search with Django ORM to store embeddings and perform semantic search. +--- + +# Integrate TiDB Vector Search with Django ORM + +This tutorial walks you through how to use [Django](https://www.djangoproject.com/) ORM to interact with the [TiDB Vector Search](/tidb-cloud/vector-search-overview.md), store embeddings, and perform vector search queries. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## Prerequisites + +To complete this tutorial, you need: + +- [Python 3.8 or higher](https://www.python.org/downloads/) installed. +- [Git](https://git-scm.com/downloads) installed. +- A TiDB Cloud Serverless cluster. Follow [creating a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create your own TiDB Cloud cluster if you don't have one. + +## Run the sample app + +You can quickly learn about how to integrate TiDB Vector Search with Django ORM by following the steps below. + +### Step 1. Clone the repository + +Clone the `tidb-vector-python` repository to your local machine: + +```shell +git clone https://github.com/pingcap/tidb-vector-python.git +``` + +### Step 2. Create a virtual environment + +Create a virtual environment for your project: + +```bash +cd tidb-vector-python/examples/orm-django-quickstart +python3 -m venv .venv +source .venv/bin/activate +``` + +### Step 3. Install required dependencies + +Install the required dependencies for the demo project: + +```bash +pip install -r requirements.txt +``` + +Alternatively, you can install the following packages for your project: + +```bash +pip install Django django-tidb mysqlclient numpy python-dotenv +``` + +If you encounter installation issues with mysqlclient, refer to the mysqlclient official documentation. + +#### What is `django-tidb` + +`django-tidb` is a TiDB dialect for Django, which enhances the Django ORM to support TiDB-specific features (for example, TiDB Vector Search) and resolves compatibility issues between TiDB and Django. + +To install `django-tidb`, choose a version that matches your Django version. For example, if you are using `django==4.2.*`, install `django-tidb==4.2.*`. The minor version does not need to be the same. It is recommended to use the latest minor version. + +For more information, refer to [django-tidb repository](https://github.com/pingcap/django-tidb). + +### Step 4. Configure the environment variables + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Ensure the configurations in the connection dialog match your operating environment. + + - **Connection Type** is set to `Public` + - **Branch** is set to `main` + - **Connect With** is set to `General` + - **Operating System** matches your environment. + + > **Tip:** + > + > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. + +4. Copy the connection parameters from the connection dialog. + + > **Tip:** + > + > If you have not set a password yet, click **Generate Password** to generate a random password. + +5. In the root directory of your Python project, create a `.env` file and paste the connection parameters to the corresponding environment variables. + + - `TIDB_HOST`: The host of the TiDB cluster. + - `TIDB_PORT`: The port of the TiDB cluster. + - `TIDB_USERNAME`: The username to connect to the TiDB cluster. + - `TIDB_PASSWORD`: The password to connect to the TiDB cluster. + - `TIDB_DATABASE`: The database name to connect to. + - `TIDB_CA_PATH`: The path to the root certificate file. + + The following is an example for macOS: + + ```dotenv + TIDB_HOST=gateway01.****.prod.aws.tidbcloud.com + TIDB_PORT=4000 + TIDB_USERNAME=********.root + TIDB_PASSWORD=******** + TIDB_DATABASE=test + TIDB_CA_PATH=/etc/ssl/cert.pem + ``` + +### Step 5. Run the demo + +Migrate the database schema: + +```bash +python manage.py migrate +``` + +Run the Django development server: + +```bash +python manage.py runserver +``` + +Open your browser and visit `http://127.0.0.1:8000` to try the demo application. Here are the available API paths: + +| API Path | Description | +| --------------------------------------- | ---------------------------------------- | +| `POST: /insert_documents` | Insert documents with embeddings. | +| `GET: /get_nearest_neighbors_documents` | Get the 3-nearest neighbor documents. | +| `GET: /get_documents_within_distance` | Get documents within a certain distance. | + +## Sample code snippets + +You can refer to the following sample code snippets to complete your own application development. + +### Connect to the TiDB cluster + +In the file `sample_project/settings.py`, add the following configurations: + +```python +dotenv.load_dotenv() + +DATABASES = { + "default": { + # https://github.com/pingcap/django-tidb + "ENGINE": "django_tidb", + "HOST": os.environ.get("TIDB_HOST", "127.0.0.1"), + "PORT": int(os.environ.get("TIDB_PORT", 4000)), + "USER": os.environ.get("TIDB_USERNAME", "root"), + "PASSWORD": os.environ.get("TIDB_PASSWORD", ""), + "NAME": os.environ.get("TIDB_DATABASE", "test"), + "OPTIONS": { + "charset": "utf8mb4", + }, + } +} + +TIDB_CA_PATH = os.environ.get("TIDB_CA_PATH", "") +if TIDB_CA_PATH: + DATABASES["default"]["OPTIONS"]["ssl_mode"] = "VERIFY_IDENTITY" + DATABASES["default"]["OPTIONS"]["ssl"] = { + "ca": TIDB_CA_PATH, + } +``` + +You can create a `.env` file in the root directory of your project and set up the environment variables `TIDB_HOST`, `TIDB_PORT`, `TIDB_USERNAME`, `TIDB_PASSWORD`, `TIDB_DATABASE`, and `TIDB_CA_PATH` with the actual values of your TiDB cluster. + +### Create vector tables + +#### Define a vector column + +`tidb-django` provides a `VectorField` to store vector embeddings in a table. + +Create a table with a column named `embedding` that stores a 3-dimensional vector. + +```python +class Document(models.Model): + content = models.TextField() + embedding = VectorField(dimensions=3) +``` + +### Store documents with embeddings + +```python +Document.objects.create(content="dog", embedding=[1, 2, 1]) +Document.objects.create(content="fish", embedding=[1, 2, 4]) +Document.objects.create(content="tree", embedding=[1, 0, 0]) +``` + +### Search the nearest neighbor documents + +TiDB Vector support the following distance functions: + +- `L1Distance` +- `L2Distance` +- `CosineDistance` +- `NegativeInnerProduct` + +Search for the top-3 documents that are semantically closest to the query vector `[1, 2, 3]` based on the cosine distance function. + +```python +results = Document.objects.annotate( + distance=CosineDistance('embedding', [1, 2, 3]) +).order_by('distance')[:3] +``` + +### Search documents within a certain distance + +Search for the documents whose cosine distance from the query vector `[1, 2, 3]` is less than 0.2. + +```python +results = Document.objects.annotate( + distance=CosineDistance('embedding', [1, 2, 3]) +).filter(distance__lt=0.2).order_by('distance')[:3] +``` + +## See also + +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) +- [Vector Search Index](/tidb-cloud/vector-search-index.md) diff --git a/tidb-cloud/vector-search-integrate-with-jinaai-embedding.md b/tidb-cloud/vector-search-integrate-with-jinaai-embedding.md new file mode 100644 index 0000000000000..71e79db40fd1d --- /dev/null +++ b/tidb-cloud/vector-search-integrate-with-jinaai-embedding.md @@ -0,0 +1,240 @@ +--- +title: Integrate TiDB Vector Search with Jina AI Embeddings API +summary: Learn how to integrate TiDB Vector Search with Jina AI Embeddings API to store embeddings and perform semantic search. +--- + +# Integrate TiDB Vector Search with Jina AI Embeddings API + +This tutorial walks you through how to use [Jina AI](https://jina.ai/) to generate embeddings for text data, and then store the embeddings in TiDB vector storage and search similar texts based on embeddings. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## Prerequisites + +To complete this tutorial, you need: + +- [Python 3.8 or higher](https://www.python.org/downloads/) installed. +- [Git](https://git-scm.com/downloads) installed. +- A TiDB Cloud Serverless cluster. Follow [creating a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create your own TiDB Cloud cluster if you don't have one. + +## Run the sample app + +You can quickly learn about how to integrate TiDB Vector Search with JinaAI Embedding by following the steps below. + +### Step 1. Clone the repository + +Clone the `tidb-vector-python` repository to your local machine: + +```shell +git clone https://github.com/pingcap/tidb-vector-python.git +``` + +### Step 2. Create a virtual environment + +Create a virtual environment for your project: + +```bash +cd tidb-vector-python/examples/jina-ai-embeddings-demo +python3 -m venv .venv +source .venv/bin/activate +``` + +### Step 3. Install required dependencies + +Install the required dependencies for the demo project: + +```bash +pip install -r requirements.txt +``` + +### Step 4. Configure the environment variables + +Get the Jina AI API key from the [Jina AI Embeddings API](https://jina.ai/embeddings/) page. Then, obtain the cluster connection string and configure environment variables as follows: + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Ensure the configurations in the connection dialog match your operating environment. + + - **Connection Type** is set to `Public` + - **Branch** is set to `main` + - **Connect With** is set to `SQLAlchemy` + - **Operating System** matches your environment. + + > **Tip:** + > + > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. + +4. Switch to the **PyMySQL** tab and click the **Copy** icon to copy the connection string. + + > **Tip:** + > + > If you have not set a password yet, click **Create password** to generate a random password. + +5. Set the Jina AI API key and the TiDB connection string as environment variables in your terminal, or create a `.env` file with the following environment variables: + + ```dotenv + JINAAI_API_KEY="****" + TIDB_DATABASE_URL="{tidb_connection_string}" + ``` + + The following is an example connection string for macOS: + + ```dotenv + TIDB_DATABASE_URL="mysql+pymysql://.root:@gateway01..prod.aws.tidbcloud.com:4000/test?ssl_ca=/etc/ssl/cert.pem&ssl_verify_cert=true&ssl_verify_identity=true" + ``` + +### Step 5. Run the demo + +```bash +python jina-ai-embeddings-demo.py +``` + +Example output: + +```text +- Inserting Data to TiDB... + - Inserting: Jina AI offers best-in-class embeddings, reranker and prompt optimizer, enabling advanced multimodal AI. + - Inserting: TiDB is an open-source MySQL-compatible database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. +- List All Documents and Their Distances to the Query: + - distance: 0.3585317326132522 + content: Jina AI offers best-in-class embeddings, reranker and prompt optimizer, enabling advanced multimodal AI. + - distance: 0.10858102967720984 + content: TiDB is an open-source MySQL-compatible database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. +- The Most Relevant Document and Its Distance to the Query: + - distance: 0.10858102967720984 + content: TiDB is an open-source MySQL-compatible database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. +``` + +## Sample code snippets + +### Get embeddings from Jina AI + +Define a `generate_embeddings` helper function to call Jina AI embeddings API: + +```python +import os +import requests +import dotenv + +dotenv.load_dotenv() + +JINAAI_API_KEY = os.getenv('JINAAI_API_KEY') + +def generate_embeddings(text: str): + JINAAI_API_URL = 'https://api.jina.ai/v1/embeddings' + JINAAI_HEADERS = { + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {JINAAI_API_KEY}' + } + JINAAI_REQUEST_DATA = { + 'input': [text], + 'model': 'jina-embeddings-v2-base-en' # with dimension 768. + } + response = requests.post(JINAAI_API_URL, headers=JINAAI_HEADERS, json=JINAAI_REQUEST_DATA) + return response.json()['data'][0]['embedding'] +``` + +### Connect to the TiDB cluster + +Connect to the TiDB cluster through SQLAlchemy: + +```python +import os +import dotenv + +from tidb_vector.sqlalchemy import VectorType +from sqlalchemy.orm import Session, declarative_base + +dotenv.load_dotenv() + +TIDB_DATABASE_URL = os.getenv('TIDB_DATABASE_URL') +assert TIDB_DATABASE_URL is not None +engine = create_engine(url=TIDB_DATABASE_URL, pool_recycle=300) +``` + +### Define the vector table schema + +Create a table named `jinaai_tidb_demo_documents` with a `content` column for storing texts and a vector column named `content_vec` for storing embeddings: + +```python +from sqlalchemy import Column, Integer, String, create_engine +from sqlalchemy.orm import declarative_base + +Base = declarative_base() + +class Document(Base): + __tablename__ = "jinaai_tidb_demo_documents" + + id = Column(Integer, primary_key=True) + content = Column(String(255), nullable=False) + content_vec = Column( + # DIMENSIONS is determined by the embedding model, + # for Jina AI's jina-embeddings-v2-base-en model it's 768. + VectorType(dim=768), + comment="hnsw(distance=cosine)" +``` + +> **Note:** +> +> - The dimension of the vector column must match the dimension of the embeddings generated by the embedding model. +> - In this example, the dimension of embeddings generated by the `jina-embeddings-v2-base-en` model is `768`. + +### Create embeddings with Jina AI and store in TiDB + +Use the Jina AI Embeddings API to generate embeddings for each piece of text and store the embeddings in TiDB: + +```python +TEXTS = [ + 'Jina AI offers best-in-class embeddings, reranker and prompt optimizer, enabling advanced multimodal AI.', + 'TiDB is an open-source MySQL-compatible database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads.', +] +data = [] + +for text in TEXTS: + # Generate embeddings for the texts via Jina AI API. + embedding = generate_embeddings(text) + data.append({ + 'text': text, + 'embedding': embedding + }) + +with Session(engine) as session: + print('- Inserting Data to TiDB...') + for item in data: + print(f' - Inserting: {item["text"]}') + session.add(Document( + content=item['text'], + content_vec=item['embedding'] + )) + session.commit() +``` + +### Perform semantic search with Jina AI embeddings in TiDB + +Generate the embedding for the query text via Jina AI embeddings API, and then search for the most relevant document based on the cosine distance between **the embedding of the query text** and **each embedding in the vector table**: + +```python +query = 'What is TiDB?' +# Generate the embedding for the query via Jina AI API. +query_embedding = generate_embeddings(query) + +with Session(engine) as session: + print('- The Most Relevant Document and Its Distance to the Query:') + doc, distance = session.query( + Document, + Document.content_vec.cosine_distance(query_embedding).label('distance') + ).order_by( + 'distance' + ).limit(1).first() + print(f' - distance: {distance}\n' + f' content: {doc.content}') +``` + +## See also + +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) +- [Vector Search Index](/tidb-cloud/vector-search-index.md) diff --git a/tidb-cloud/vector-search-integrate-with-langchain.md b/tidb-cloud/vector-search-integrate-with-langchain.md new file mode 100644 index 0000000000000..04b518a82b31d --- /dev/null +++ b/tidb-cloud/vector-search-integrate-with-langchain.md @@ -0,0 +1,586 @@ +--- +title: Integrate Vector Search with LangChain +summary: Learn how to integrate Vector Search in TiDB Cloud with LangChain. +--- + +# Integrate Vector Search with LangChain + +This tutorial demonstrates how to integrate the [vector search](/tidb-cloud/vector-search-overview.md) feature in TiDB Cloud with [LangChain](https://python.langchain.com/). + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +> **Tip** +> +> You can view the complete [sample code](https://github.com/langchain-ai/langchain/blob/master/docs/docs/integrations/vectorstores/tidb_vector.ipynb) on Jupyter Notebook, or run the sample code directly in the [Colab](https://colab.research.google.com/github/langchain-ai/langchain/blob/master/docs/docs/integrations/vectorstores/tidb_vector.ipynb) online environment. + +## Prerequisites + +To complete this tutorial, you need: + +- [Python 3.8 or higher](https://www.python.org/downloads/) installed. +- [Jupyter Notebook](https://jupyter.org/install) installed. +- [Git](https://git-scm.com/downloads) installed. +- A TiDB Cloud Serverless cluster. Follow [creating a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create your own TiDB Cloud cluster if you don't have one. + +## Get started + +This section provides step-by-step instructions for integrating TiDB Vector Search with LangChain to perform semantic searches. + +### Step 1. Create a new Jupyter Notebook file + +In your preferred directory, create a new Jupyter Notebook file named `integrate_with_langchain.ipynb`: + +```shell +touch integrate_with_langchain.ipynb +``` + +### Step 2. Install required dependencies + +In your project directory, run the following command to install the required packages: + +```shell +!pip install langchain langchain-community +!pip install langchain-openai +!pip install pymysql +!pip install tidb-vector +``` + +Open the `integrate_with_langchain.ipynb` file in Jupyter Notebook, and then add the following code to import the required packages: + +```python +from langchain_community.document_loaders import TextLoader +from langchain_community.vectorstores import TiDBVectorStore +from langchain_openai import OpenAIEmbeddings +from langchain_text_splitters import CharacterTextSplitter +``` + +### Step 3. Set up your environment + +Take the following steps to obtain the cluster connection string and configure environment variables: + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Ensure the configurations in the connection dialog match your operating environment. + + - **Connection Type** is set to `Public`. + - **Branch** is set to `main`. + - **Connect With** is set to `SQLAlchemy`. + - **Operating System** matches your environment. + +4. Click the **PyMySQL** tab and copy the connection string. + + > **Tip:** + > + > If you have not set a password yet, click **Generate Password** to generate a random password. + +5. Configure environment variables. + + This document uses [OpenAI](https://platform.openai.com/docs/introduction) as the embedding model provider. In this step, you need to provide the connection string obtained from the previous step and your [OpenAI API key](https://platform.openai.com/docs/quickstart/step-2-set-up-your-api-key). + + To configure the environment variables, run the following code. You will be prompted to enter your connection string and OpenAI API key: + + ```python + # Use getpass to securely prompt for environment variables in your terminal. + import getpass + import os + + # Copy your connection string from the TiDB Cloud console. + # Connection string format: "mysql+pymysql://:@:4000/?ssl_ca=/etc/ssl/cert.pem&ssl_verify_cert=true&ssl_verify_identity=true" + tidb_connection_string = getpass.getpass("TiDB Connection String:") + os.environ["OPENAI_API_KEY"] = getpass.getpass("OpenAI API Key:") + ``` + +### Step 4. Load the sample document + +#### Step 4.1 Download the sample document + +In your project directory, create a directory named `data/how_to/` and download the sample document [`state_of_the_union.txt`](https://github.com/langchain-ai/langchain/blob/master/docs/docs/how_to/state_of_the_union.txt) from the [langchain-ai/langchain](https://github.com/langchain-ai/langchain) GitHub repository. + +```shell +!mkdir -p 'data/how_to/' +!wget 'https://raw.githubusercontent.com/langchain-ai/langchain/master/docs/docs/how_to/state_of_the_union.txt' -O 'data/how_to/state_of_the_union.txt' +``` + +#### Step 4.2 Load and split the document + +Load the sample document from `data/how_to/state_of_the_union.txt` and split it into chunks of approximately 1,000 characters each using a `CharacterTextSplitter`. + +```python +loader = TextLoader("data/how_to/state_of_the_union.txt") +documents = loader.load() +text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0) +docs = text_splitter.split_documents(documents) +``` + +### Step 5. Embed and store document vectors + +TiDB vector store supports both cosine distance (`consine`) and Euclidean distance (`l2`) for measuring similarity between vectors. The default strategy is cosine distance. + +The following code creates a table named `embedded_documents` in TiDB, which is optimized for vector search. + +```python +embeddings = OpenAIEmbeddings() +vector_store = TiDBVectorStore.from_documents( + documents=docs, + embedding=embeddings, + table_name="embedded_documents", + connection_string=tidb_connection_string, + distance_strategy="cosine", # default, another option is "l2" +) +``` + +Upon successful execution, you can directly view and access the `embedded_documents` table in your TiDB database. + +### Step 6. Perform a vector search + +This step demonstrates how to query "What did the president say about Ketanji Brown Jackson" from the document `state_of_the_union.txt`. + +```python +query = "What did the president say about Ketanji Brown Jackson" +``` + +#### Option 1: Use `similarity_search_with_score()` + +The `similarity_search_with_score()` method calculates the vector space distance between the documents and the query. This distance serves as a similarity score, determined by the chosen `distance_strategy`. The method returns the top `k` documents with the lowest scores. A lower score indicates a higher similarity between a document and your query. + +```python +docs_with_score = vector_store.similarity_search_with_score(query, k=3) +for doc, score in docs_with_score: + print("-" * 80) + print("Score: ", score) + print(doc.page_content) + print("-" * 80) +``` + +
    + Expected output + +```plain +-------------------------------------------------------------------------------- +Score: 0.18472413652518527 +Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. + +Tonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. + +One of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. + +And I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence. +-------------------------------------------------------------------------------- +-------------------------------------------------------------------------------- +Score: 0.21757513022785557 +A former top litigator in private practice. A former federal public defender. And from a family of public school educators and police officers. A consensus builder. Since she’s been nominated, she’s received a broad range of support—from the Fraternal Order of Police to former judges appointed by Democrats and Republicans. + +And if we are to advance liberty and justice, we need to secure the Border and fix the immigration system. + +We can do both. At our border, we’ve installed new technology like cutting-edge scanners to better detect drug smuggling. + +We’ve set up joint patrols with Mexico and Guatemala to catch more human traffickers. + +We’re putting in place dedicated immigration judges so families fleeing persecution and violence can have their cases heard faster. + +We’re securing commitments and supporting partners in South and Central America to host more refugees and secure their own borders. +-------------------------------------------------------------------------------- +-------------------------------------------------------------------------------- +Score: 0.22676987253721725 +And for our LGBTQ+ Americans, let’s finally get the bipartisan Equality Act to my desk. The onslaught of state laws targeting transgender Americans and their families is wrong. + +As I said last year, especially to our younger transgender Americans, I will always have your back as your President, so you can be yourself and reach your God-given potential. + +While it often appears that we never agree, that isn’t true. I signed 80 bipartisan bills into law last year. From preventing government shutdowns to protecting Asian-Americans from still-too-common hate crimes to reforming military justice. + +And soon, we’ll strengthen the Violence Against Women Act that I first wrote three decades ago. It is important for us to show the nation that we can come together and do big things. + +So tonight I’m offering a Unity Agenda for the Nation. Four big things we can do together. + +First, beat the opioid epidemic. +-------------------------------------------------------------------------------- +``` + +
    + +#### Option 2: Use `similarity_search_with_relevance_scores()` + +The `similarity_search_with_relevance_scores()` method returns the top `k` documents with the highest relevance scores. A higher score indicates a higher degree of similarity between a document and your query. + +```python +docs_with_relevance_score = vector_store.similarity_search_with_relevance_scores(query, k=2) +for doc, score in docs_with_relevance_score: + print("-" * 80) + print("Score: ", score) + print(doc.page_content) + print("-" * 80) +``` + +
    + Expected output + +```plain +-------------------------------------------------------------------------------- +Score: 0.8152758634748147 +Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. + +Tonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. + +One of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. + +And I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence. +-------------------------------------------------------------------------------- +-------------------------------------------------------------------------------- +Score: 0.7824248697721444 +A former top litigator in private practice. A former federal public defender. And from a family of public school educators and police officers. A consensus builder. Since she’s been nominated, she’s received a broad range of support—from the Fraternal Order of Police to former judges appointed by Democrats and Republicans. + +And if we are to advance liberty and justice, we need to secure the Border and fix the immigration system. + +We can do both. At our border, we’ve installed new technology like cutting-edge scanners to better detect drug smuggling. + +We’ve set up joint patrols with Mexico and Guatemala to catch more human traffickers. + +We’re putting in place dedicated immigration judges so families fleeing persecution and violence can have their cases heard faster. + +We’re securing commitments and supporting partners in South and Central America to host more refugees and secure their own borders. +-------------------------------------------------------------------------------- +``` + +
    + +### Use as a retriever + +In Langchain, a [retriever](https://python.langchain.com/v0.2/docs/concepts/#retrievers) is an interface that retrieves documents in response to an unstructured query, providing more functionality than a vector store. The following code demonstrates how to use TiDB vector store as a retriever. + +```python +retriever = vector_store.as_retriever( + search_type="similarity_score_threshold", + search_kwargs={"k": 3, "score_threshold": 0.8}, +) +docs_retrieved = retriever.invoke(query) +for doc in docs_retrieved: + print("-" * 80) + print(doc.page_content) + print("-" * 80) +``` + +The expected output is as follows: + +``` +-------------------------------------------------------------------------------- +Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. + +Tonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. + +One of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. + +And I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence. +-------------------------------------------------------------------------------- +``` + +### Remove the vector store + +To remove an existing TiDB vector store, use the `drop_vectorstore()` method: + +```python +vector_store.drop_vectorstore() +``` + +## Search with metadata filters + +To refine your searches, you can use metadata filters to retrieve specific nearest-neighbor results that match the applied filters. + +### Supported metadata types + +Each document in the TiDB vector store can be paired with metadata, structured as key-value pairs within a JSON object. Keys are always strings, while values can be any of the following types: + +- String +- Number: integer or floating point +- Boolean: `true` or `false` + +For example, the following is a valid metadata payload: + +```json +{ + "page": 12, + "book_title": "Siddhartha" +} +``` + +### Metadata filter syntax + +Available filters include the following: + +- `$or`: Selects vectors that match any one of the specified conditions. +- `$and`: Selects vectors that match all the specified conditions. +- `$eq`: Equal to the specified value. +- `$ne`: Not equal to the specified value. +- `$gt`: Greater than the specified value. +- `$gte`: Greater than or equal to the specified value. +- `$lt`: Less than the specified value. +- `$lte`: Less than or equal to the specified value. +- `$in`: In the specified array of values. +- `$nin`: Not in the specified array of values. + +If the metadata of a document is as follows: + +```json +{ + "page": 12, + "book_title": "Siddhartha" +} +``` + +The following metadata filters can match this document: + +```json +{ "page": 12 } +``` + +```json +{ "page": { "$eq": 12 } } +``` + +```json +{ + "page": { + "$in": [11, 12, 13] + } +} +``` + +```json +{ "page": { "$nin": [13] } } +``` + +```json +{ "page": { "$lt": 11 } } +``` + +```json +{ + "$or": [{ "page": 11 }, { "page": 12 }], + "$and": [{ "page": 12 }, { "page": 13 }] +} +``` + +In a metadata filter, TiDB treats each key-value pair as a separate filter clause and combines these clauses using the `AND` logical operator. + +### Example + +The following example adds two documents to `TiDBVectorStore` and adds a `title` field to each document as the metadata: + +```python +vector_store.add_texts( + texts=[ + "TiDB Vector offers advanced, high-speed vector processing capabilities, enhancing AI workflows with efficient data handling and analytics support.", + "TiDB Vector, starting as low as $10 per month for basic usage", + ], + metadatas=[ + {"title": "TiDB Vector functionality"}, + {"title": "TiDB Vector Pricing"}, + ], +) +``` + +The expected output is as follows: + +```plain +[UUID('c782cb02-8eec-45be-a31f-fdb78914f0a7'), + UUID('08dcd2ba-9f16-4f29-a9b7-18141f8edae3')] +``` + +Perform a similarity search with metadata filters: + +```python +docs_with_score = vector_store.similarity_search_with_score( + "Introduction to TiDB Vector", filter={"title": "TiDB Vector functionality"}, k=4 +) +for doc, score in docs_with_score: + print("-" * 80) + print("Score: ", score) + print(doc.page_content) + print("-" * 80) +``` + +The expected output is as follows: + +```plain +-------------------------------------------------------------------------------- +Score: 0.12761409169211535 +TiDB Vector offers advanced, high-speed vector processing capabilities, enhancing AI workflows with efficient data handling and analytics support. +-------------------------------------------------------------------------------- +``` + +## Advanced usage example: travel agent + +This section demonstrates a use case of integrating vector search with Langchain for a travel agent. The goal is to create personalized travel reports for clients, helping them find airports with specific amenities, such as clean lounges and vegetarian options. + +The process involves two main steps: + +1. Perform a semantic search across airport reviews to identify airport codes that match the desired amenities. +2. Execute a SQL query to merge these codes with route information, highlighting airlines and destinations that align with user's preferences. + +### Prepare data + +First, create a table to store airport route data: + +```python +# Create a table to store flight plan data. +vector_store.tidb_vector_client.execute( + """CREATE TABLE airplan_routes ( + id INT AUTO_INCREMENT PRIMARY KEY, + airport_code VARCHAR(10), + airline_code VARCHAR(10), + destination_code VARCHAR(10), + route_details TEXT, + duration TIME, + frequency INT, + airplane_type VARCHAR(50), + price DECIMAL(10, 2), + layover TEXT + );""" +) + +# Insert some sample data into airplan_routes and the vector table. +vector_store.tidb_vector_client.execute( + """INSERT INTO airplan_routes ( + airport_code, + airline_code, + destination_code, + route_details, + duration, + frequency, + airplane_type, + price, + layover + ) VALUES + ('JFK', 'DL', 'LAX', 'Non-stop from JFK to LAX.', '06:00:00', 5, 'Boeing 777', 299.99, 'None'), + ('LAX', 'AA', 'ORD', 'Direct LAX to ORD route.', '04:00:00', 3, 'Airbus A320', 149.99, 'None'), + ('EFGH', 'UA', 'SEA', 'Daily flights from SFO to SEA.', '02:30:00', 7, 'Boeing 737', 129.99, 'None'); + """ +) +vector_store.add_texts( + texts=[ + "Clean lounges and excellent vegetarian dining options. Highly recommended.", + "Comfortable seating in lounge areas and diverse food selections, including vegetarian.", + "Small airport with basic facilities.", + ], + metadatas=[ + {"airport_code": "JFK"}, + {"airport_code": "LAX"}, + {"airport_code": "EFGH"}, + ], +) +``` + +The expected output is as follows: + +```plain +[UUID('6dab390f-acd9-4c7d-b252-616606fbc89b'), + UUID('9e811801-0e6b-4893-8886-60f4fb67ce69'), + UUID('f426747c-0f7b-4c62-97ed-3eeb7c8dd76e')] +``` + +### Perform a semantic search + +The following code searches for airports with clean facilities and vegetarian options: + +```python +retriever = vector_store.as_retriever( + search_type="similarity_score_threshold", + search_kwargs={"k": 3, "score_threshold": 0.85}, +) +semantic_query = "Could you recommend a US airport with clean lounges and good vegetarian dining options?" +reviews = retriever.invoke(semantic_query) +for r in reviews: + print("-" * 80) + print(r.page_content) + print(r.metadata) + print("-" * 80) +``` + +The expected output is as follows: + +```plain +-------------------------------------------------------------------------------- +Clean lounges and excellent vegetarian dining options. Highly recommended. +{'airport_code': 'JFK'} +-------------------------------------------------------------------------------- +-------------------------------------------------------------------------------- +Comfortable seating in lounge areas and diverse food selections, including vegetarian. +{'airport_code': 'LAX'} +-------------------------------------------------------------------------------- +``` + +### Retrieve detailed airport information + +Extract airport codes from the search results and query the database for detailed route information: + +```python +# Extracting airport codes from the metadata +airport_codes = [review.metadata["airport_code"] for review in reviews] + +# Executing a query to get the airport details +search_query = "SELECT * FROM airplan_routes WHERE airport_code IN :codes" +params = {"codes": tuple(airport_codes)} + +airport_details = vector_store.tidb_vector_client.execute(search_query, params) +airport_details.get("result") +``` + +The expected output is as follows: + +```plain +[(1, 'JFK', 'DL', 'LAX', 'Non-stop from JFK to LAX.', datetime.timedelta(seconds=21600), 5, 'Boeing 777', Decimal('299.99'), 'None'), + (2, 'LAX', 'AA', 'ORD', 'Direct LAX to ORD route.', datetime.timedelta(seconds=14400), 3, 'Airbus A320', Decimal('149.99'), 'None')] +``` + +### Streamline the process + +Alternatively, you can streamline the entire process using a single SQL query: + +```python +search_query = f""" + SELECT + VEC_Cosine_Distance(se.embedding, :query_vector) as distance, + ar.*, + se.document as airport_review + FROM + airplan_routes ar + JOIN + {TABLE_NAME} se ON ar.airport_code = JSON_UNQUOTE(JSON_EXTRACT(se.meta, '$.airport_code')) + ORDER BY distance ASC + LIMIT 5; +""" +query_vector = embeddings.embed_query(semantic_query) +params = {"query_vector": str(query_vector)} +airport_details = vector_store.tidb_vector_client.execute(search_query, params) +airport_details.get("result") +``` + +The expected output is as follows: + +```plain +[(0.1219207353407008, 1, 'JFK', 'DL', 'LAX', 'Non-stop from JFK to LAX.', datetime.timedelta(seconds=21600), 5, 'Boeing 777', Decimal('299.99'), 'None', 'Clean lounges and excellent vegetarian dining options. Highly recommended.'), + (0.14613754359804654, 2, 'LAX', 'AA', 'ORD', 'Direct LAX to ORD route.', datetime.timedelta(seconds=14400), 3, 'Airbus A320', Decimal('149.99'), 'None', 'Comfortable seating in lounge areas and diverse food selections, including vegetarian.'), + (0.19840519342700513, 3, 'EFGH', 'UA', 'SEA', 'Daily flights from SFO to SEA.', datetime.timedelta(seconds=9000), 7, 'Boeing 737', Decimal('129.99'), 'None', 'Small airport with basic facilities.')] +``` + +### Clean up data + +Finally, clean up the resources by dropping the created table: + +```python +vector_store.tidb_vector_client.execute("DROP TABLE airplan_routes") +``` + +The expected output is as follows: + +```plain +{'success': True, 'result': 0, 'error': None} +``` + +## See also + +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) +- [Vector Search Index](/tidb-cloud/vector-search-index.md) diff --git a/tidb-cloud/vector-search-integrate-with-llamaindex.md b/tidb-cloud/vector-search-integrate-with-llamaindex.md new file mode 100644 index 0000000000000..117afbbecf9b0 --- /dev/null +++ b/tidb-cloud/vector-search-integrate-with-llamaindex.md @@ -0,0 +1,266 @@ +--- +title: Integrate Vector Search with LlamaIndex +summary: Learn how to integrate TiDB Vector Search with LlamaIndex. +--- + +# Integrate Vector Search with LlamaIndex + +This tutorial demonstrates how to integrate the [vector search](/tidb-cloud/vector-search-overview.md) feature of TiDB with [LlamaIndex](https://www.llamaindex.ai). + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +> **Tip** +> +> You can view the complete [sample code](https://github.com/run-llama/llama_index/blob/main/docs/docs/examples/vector_stores/TiDBVector.ipynb) on Jupyter Notebook, or run the sample code directly in the [Colab](https://colab.research.google.com/github/run-llama/llama_index/blob/main/docs/docs/examples/vector_stores/TiDBVector.ipynb) online environment. + +## Prerequisites + +To complete this tutorial, you need: + +- [Python 3.8 or higher](https://www.python.org/downloads/) installed. +- [Jupyter Notebook](https://jupyter.org/install) installed. +- [Git](https://git-scm.com/downloads) installed. +- A TiDB Cloud Serverless cluster. Follow [creating a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create your own TiDB Cloud cluster if you don't have one. + +## Get started + +This section provides step-by-step instructions for integrating TiDB Vector Search with LlamaIndex to perform semantic searches. + +### Step 1. Create a new Jupyter Notebook file + +In the root directory, create a new Jupyter Notebook file named `integrate_with_llamaindex.ipynb`: + +```shell +touch integrate_with_llamaindex.ipynb +``` + +### Step 2. Install required dependencies + +In your project directory, run the following command to install the required packages: + +```shell +pip install llama-index-vector-stores-tidbvector +pip install llama-index +``` + +Open the `integrate_with_llamaindex.ipynb` file in Jupyter Notebook and add the following code to import the required packages: + +```python +import textwrap + +from llama_index.core import SimpleDirectoryReader, StorageContext +from llama_index.core import VectorStoreIndex +from llama_index.vector_stores.tidbvector import TiDBVectorStore +``` + +### Step 3. Configure environment variables + +Take the following steps to obtain the cluster connection string and configure environment variables: + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Ensure the configurations in the connection dialog match your operating environment. + + - **Connection Type** is set to `Public`. + - **Branch** is set to `main`. + - **Connect With** is set to `SQLAlchemy`. + - **Operating System** matches your environment. + +4. Click the **PyMySQL** tab and copy the connection string. + + > **Tip:** + > + > If you have not set a password yet, click **Generate Password** to generate a random password. + +5. Configure environment variables. + + This document uses [OpenAI](https://platform.openai.com/docs/introduction) as the embedding model provider. In this step, you need to provide the connection string obtained from from the previous step and your [OpenAI API key](https://platform.openai.com/docs/quickstart/step-2-set-up-your-api-key). + + To configure the environment variables, run the following code. You will be prompted to enter your connection string and OpenAI API key: + + ```python + # Use getpass to securely prompt for environment variables in your terminal. + import getpass + import os + + # Copy your connection string from the TiDB Cloud console. + # Connection string format: "mysql+pymysql://:@:4000/?ssl_ca=/etc/ssl/cert.pem&ssl_verify_cert=true&ssl_verify_identity=true" + tidb_connection_string = getpass.getpass("TiDB Connection String:") + os.environ["OPENAI_API_KEY"] = getpass.getpass("OpenAI API Key:") + ``` + +### Step 4. Load the sample document + +#### Step 4.1 Download the sample document + +In your project directory, create a directory named `data/paul_graham/` and download the sample document [`paul_graham_essay.txt`](https://github.com/run-llama/llama_index/blob/main/docs/docs/examples/data/paul_graham/paul_graham_essay.txt) from the [run-llama/llama_index](https://github.com/run-llama/llama_index) GitHub repository. + +```shell +!mkdir -p 'data/paul_graham/' +!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/paul_graham/paul_graham_essay.txt' -O 'data/paul_graham/paul_graham_essay.txt' +``` + +#### Step 4.2 Load the document + +Load the sample document from `data/paul_graham/paul_graham_essay.txt` using the `SimpleDirectoryReader` class. + +```python +documents = SimpleDirectoryReader("./data/paul_graham").load_data() +print("Document ID:", documents[0].doc_id) + +for index, document in enumerate(documents): + document.metadata = {"book": "paul_graham"} +``` + +### Step 5. Embed and store document vectors + +#### Step 5.1 Initialize the TiDB vector store + +The following code creates a table named `paul_graham_test` in TiDB, which is optimized for vector search. + +```python +tidbvec = TiDBVectorStore( + connection_string=tidb_connection_url, + table_name="paul_graham_test", + distance_strategy="cosine", + vector_dimension=1536, + drop_existing_table=False, +) +``` + +Upon successful execution, you can directly view and access the `paul_graham_test` table in your TiDB database. + +#### Step 5.2 Generate and store embeddings + +The following code parses the documents, generates embeddings, and stores them in the TiDB vector store. + +```python +storage_context = StorageContext.from_defaults(vector_store=tidbvec) +index = VectorStoreIndex.from_documents( + documents, storage_context=storage_context, show_progress=True +) +``` + +The expected output is as follows: + +```plain +Parsing nodes: 100%|██████████| 1/1 [00:00<00:00, 8.76it/s] +Generating embeddings: 100%|██████████| 21/21 [00:02<00:00, 8.22it/s] +``` + +### Step 6. Perform a vector search + +The following creates a query engine based on the TiDB vector store and performs a semantic similarity search. + +```python +query_engine = index.as_query_engine() +response = query_engine.query("What did the author do?") +print(textwrap.fill(str(response), 100)) +``` + +> **Note** +> +> `TiDBVectorStore` only supports the [`default`](https://docs.llamaindex.ai/en/stable/api_reference/storage/vector_store/?h=vectorstorequerymode#llama_index.core.vector_stores.types.VectorStoreQueryMode) query mode. + +The expected output is as follows: + +```plain +The author worked on writing, programming, building microcomputers, giving talks at conferences, +publishing essays online, developing spam filters, painting, hosting dinner parties, and purchasing +a building for office use. +``` + +### Step 7. Search with metadata filters + +To refine your searches, you can use metadata filters to retrieve specific nearest-neighbor results that match the applied filters. + +#### Query with `book != "paul_graham"` filter + +The following example excludes results where the `book` metadata field is `"paul_graham"`: + +```python +from llama_index.core.vector_stores.types import ( + MetadataFilter, + MetadataFilters, +) + +query_engine = index.as_query_engine( + filters=MetadataFilters( + filters=[ + MetadataFilter(key="book", value="paul_graham", operator="!="), + ] + ), + similarity_top_k=2, +) +response = query_engine.query("What did the author learn?") +print(textwrap.fill(str(response), 100)) +``` + +The expected output is as follows: + +```plain +Empty Response +``` + +#### Query with `book == "paul_graham"` filter + +The following example filters results to include only documents where the `book` metadata field is `"paul_graham"`: + +```python +from llama_index.core.vector_stores.types import ( + MetadataFilter, + MetadataFilters, +) + +query_engine = index.as_query_engine( + filters=MetadataFilters( + filters=[ + MetadataFilter(key="book", value="paul_graham", operator="=="), + ] + ), + similarity_top_k=2, +) +response = query_engine.query("What did the author learn?") +print(textwrap.fill(str(response), 100)) +``` + +The expected output is as follows: + +```plain +The author learned programming on an IBM 1401 using an early version of Fortran in 9th grade, then +later transitioned to working with microcomputers like the TRS-80 and Apple II. Additionally, the +author studied philosophy in college but found it unfulfilling, leading to a switch to studying AI. +Later on, the author attended art school in both the US and Italy, where they observed a lack of +substantial teaching in the painting department. +``` + +### Step 8. Delete documents + +Delete the first document from the index: + +```python +tidbvec.delete(documents[0].doc_id) +``` + +Check whether the documents had been deleted: + +```python +query_engine = index.as_query_engine() +response = query_engine.query("What did the author learn?") +print(textwrap.fill(str(response), 100)) +``` + +The expected output is as follows: + +```plain +Empty Response +``` + +## See also + +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) +- [Vector Search Index](/tidb-cloud/vector-search-index.md) diff --git a/tidb-cloud/vector-search-integrate-with-peewee.md b/tidb-cloud/vector-search-integrate-with-peewee.md new file mode 100644 index 0000000000000..0a72a34135ef8 --- /dev/null +++ b/tidb-cloud/vector-search-integrate-with-peewee.md @@ -0,0 +1,211 @@ +--- +title: Integrate TiDB Vector Search with peewee +summary: Learn how to integrate TiDB Vector Search with peewee to store embeddings and perform semantic searches. +--- + +# Integrate TiDB Vector Search with peewee + +This tutorial walks you through how to use [peewee](https://docs.peewee-orm.com/) to interact with the [TiDB Vector Search](/tidb-cloud/vector-search-overview.md), store embeddings, and perform vector search queries. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## Prerequisites + +To complete this tutorial, you need: + +- [Python 3.8 or higher](https://www.python.org/downloads/) installed. +- [Git](https://git-scm.com/downloads) installed. +- A TiDB Cloud Serverless cluster. Follow [creating a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create your own TiDB Cloud cluster if you don't have one. + +## Run the sample app + +You can quickly learn about how to integrate TiDB Vector Search with peewee by following the steps below. + +### Step 1. Clone the repository + +Clone the [`tidb-vector-python`](https://github.com/pingcap/tidb-vector-python) repository to your local machine: + +```shell +git clone https://github.com/pingcap/tidb-vector-python.git +``` + +### Step 2. Create a virtual environment + +Create a virtual environment for your project: + +```bash +cd tidb-vector-python/examples/orm-peewee-quickstart +python3 -m venv .venv +source .venv/bin/activate +``` + +### Step 3. Install required dependencies + +Install the required dependencies for the demo project: + +```bash +pip install -r requirements.txt +``` + +Alternatively, you can install the following packages for your project: + +```bash +pip install peewee pymysql python-dotenv tidb-vector +``` + +### Step 4. Configure the environment variables + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Ensure the configurations in the connection dialog match your operating environment. + + - **Connection Type** is set to `Public`. + - **Branch** is set to `main`. + - **Connect With** is set to `General`. + - **Operating System** matches your environment. + + > **Tip:** + > + > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. + +4. Copy the connection parameters from the connection dialog. + + > **Tip:** + > + > If you have not set a password yet, click **Generate Password** to generate a random password. + +5. In the root directory of your Python project, create a `.env` file and paste the connection parameters to the corresponding environment variables. + + - `TIDB_HOST`: The host of the TiDB cluster. + - `TIDB_PORT`: The port of the TiDB cluster. + - `TIDB_USERNAME`: The username to connect to the TiDB cluster. + - `TIDB_PASSWORD`: The password to connect to the TiDB cluster. + - `TIDB_DATABASE`: The database name to connect to. + - `TIDB_CA_PATH`: The path to the root certificate file. + + The following is an example for macOS: + + ```dotenv + TIDB_HOST=gateway01.****.prod.aws.tidbcloud.com + TIDB_PORT=4000 + TIDB_USERNAME=********.root + TIDB_PASSWORD=******** + TIDB_DATABASE=test + TIDB_CA_PATH=/etc/ssl/cert.pem + ``` + +### Step 5. Run the demo + +```bash +python peewee-quickstart.py +``` + +Example output: + +```text +Get 3-nearest neighbor documents: + - distance: 0.00853986601633272 + document: fish + - distance: 0.12712843905603044 + document: dog + - distance: 0.7327387580875756 + document: tree +Get documents within a certain distance: + - distance: 0.00853986601633272 + document: fish + - distance: 0.12712843905603044 + document: dog +``` + +## Sample code snippets + +You can refer to the following sample code snippets to develop your application. + +### Create vector tables + +#### Connect to TiDB cluster + +```python +import os +import dotenv + +from peewee import Model, MySQLDatabase, SQL, TextField +from tidb_vector.peewee import VectorField + +dotenv.load_dotenv() + +# Using `pymysql` as the driver. +connect_kwargs = { + 'ssl_verify_cert': True, + 'ssl_verify_identity': True, +} + +# Using `mysqlclient` as the driver. +# connect_kwargs = { +# 'ssl_mode': 'VERIFY_IDENTITY', +# 'ssl': { +# # Root certificate default path +# # https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters/#root-certificate-default-path +# 'ca': os.environ.get('TIDB_CA_PATH', '/path/to/ca.pem'), +# }, +# } + +db = MySQLDatabase( + database=os.environ.get('TIDB_DATABASE', 'test'), + user=os.environ.get('TIDB_USERNAME', 'root'), + password=os.environ.get('TIDB_PASSWORD', ''), + host=os.environ.get('TIDB_HOST', 'localhost'), + port=int(os.environ.get('TIDB_PORT', '4000')), + **connect_kwargs, +) +``` + +#### Define a vector column + +Create a table with a column named `peewee_demo_documents` that stores a 3-dimensional vector. + +```python +class Document(Model): + class Meta: + database = db + table_name = 'peewee_demo_documents' + + content = TextField() + embedding = VectorField(3) +``` + +### Store documents with embeddings + +```python +Document.create(content='dog', embedding=[1, 2, 1]) +Document.create(content='fish', embedding=[1, 2, 4]) +Document.create(content='tree', embedding=[1, 0, 0]) +``` + +### Search the nearest neighbor documents + +Search for the top-3 documents that are semantically closest to the query vector `[1, 2, 3]` based on the cosine distance function. + +```python +distance = Document.embedding.cosine_distance([1, 2, 3]).alias('distance') +results = Document.select(Document, distance).order_by(distance).limit(3) +``` + +### Search documents within a certain distance + +Search for the documents whose cosine distance from the query vector `[1, 2, 3]` is less than 0.2. + +```python +distance_expression = Document.embedding.cosine_distance([1, 2, 3]) +distance = distance_expression.alias('distance') +results = Document.select(Document, distance).where(distance_expression < 0.2).order_by(distance).limit(3) +``` + +## See also + +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) +- [Vector Search Index](/tidb-cloud/vector-search-index.md) diff --git a/tidb-cloud/vector-search-integrate-with-sqlalchemy.md b/tidb-cloud/vector-search-integrate-with-sqlalchemy.md new file mode 100644 index 0000000000000..f9fea8e3f11cd --- /dev/null +++ b/tidb-cloud/vector-search-integrate-with-sqlalchemy.md @@ -0,0 +1,185 @@ +--- +title: Integrate TiDB Vector Search with SQLAlchemy +summary: Learn how to integrate TiDB Vector Search with SQLAlchemy to store embeddings and perform semantic searches. +--- + +# Integrate TiDB Vector Search with SQLAlchemy + +This tutorial walks you through how to use [SQLAlchemy](https://www.sqlalchemy.org/) to interact with [TiDB Vector Search](/tidb-cloud/vector-search-overview.md), store embeddings, and perform vector search queries. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## Prerequisites + +To complete this tutorial, you need: + +- [Python 3.8 or higher](https://www.python.org/downloads/) installed. +- [Git](https://git-scm.com/downloads) installed. +- A TiDB Cloud Serverless cluster. Follow [creating a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create your own TiDB Cloud cluster if you don't have one. + +## Run the sample app + +You can quickly learn about how to integrate TiDB Vector Search with SQLAlchemy by following the steps below. + +### Step 1. Clone the repository + +Clone the `tidb-vector-python` repository to your local machine: + +```shell +git clone https://github.com/pingcap/tidb-vector-python.git +``` + +### Step 2. Create a virtual environment + +Create a virtual environment for your project: + +```bash +cd tidb-vector-python/examples/orm-sqlalchemy-quickstart +python3 -m venv .venv +source .venv/bin/activate +``` + +### Step 3. Install the required dependencies + +Install the required dependencies for the demo project: + +```bash +pip install -r requirements.txt +``` + +Alternatively, you can install the following packages for your project: + +```bash +pip install pymysql python-dotenv sqlalchemy tidb-vector +``` + +### Step 4. Configure the environment variables + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Ensure the configurations in the connection dialog match your environment. + + - **Connection Type** is set to `Public`. + - **Branch** is set to `main`. + - **Connect With** is set to `SQLAlchemy`. + - **Operating System** matches your environment. + + > **Tip:** + > + > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. + +4. Click the **PyMySQL** tab and copy the connection string. + + > **Tip:** + > + > If you have not set a password yet, click **Generate Password** to generate a random password. + +5. In the root directory of your Python project, create a `.env` file and paste the connection string into it. + + The following is an example for macOS: + + ```dotenv + TIDB_DATABASE_URL="mysql+pymysql://.root:@gateway01..prod.aws.tidbcloud.com:4000/test?ssl_ca=/etc/ssl/cert.pem&ssl_verify_cert=true&ssl_verify_identity=true" + ``` + +### Step 5. Run the demo + +```bash +python sqlalchemy-quickstart.py +``` + +Example output: + +```text +Get 3-nearest neighbor documents: + - distance: 0.00853986601633272 + document: fish + - distance: 0.12712843905603044 + document: dog + - distance: 0.7327387580875756 + document: tree +Get documents within a certain distance: + - distance: 0.00853986601633272 + document: fish + - distance: 0.12712843905603044 + document: dog +``` + +## Sample code snippets + +You can refer to the following sample code snippets to develop your application. + +### Create vector tables + +#### Connect to TiDB cluster + +```python +import os +import dotenv + +from sqlalchemy import Column, Integer, create_engine, Text +from sqlalchemy.orm import declarative_base, Session +from tidb_vector.sqlalchemy import VectorType + +dotenv.load_dotenv() + +tidb_connection_string = os.environ['TIDB_DATABASE_URL'] +engine = create_engine(tidb_connection_string) +``` + +#### Define a vector column + +Create a table with a column named `embedding` that stores a 3-dimensional vector. + +```python +Base = declarative_base() + +class Document(Base): + __tablename__ = 'sqlalchemy_demo_documents' + id = Column(Integer, primary_key=True) + content = Column(Text) + embedding = Column(VectorType(3)) +``` + +### Store documents with embeddings + +```python +with Session(engine) as session: + session.add(Document(content="dog", embedding=[1, 2, 1])) + session.add(Document(content="fish", embedding=[1, 2, 4])) + session.add(Document(content="tree", embedding=[1, 0, 0])) + session.commit() +``` + +### Search the nearest neighbor documents + +Search for the top-3 documents that are semantically closest to the query vector `[1, 2, 3]` based on the cosine distance function. + +```python +with Session(engine) as session: + distance = Document.embedding.cosine_distance([1, 2, 3]).label('distance') + results = session.query( + Document, distance + ).order_by(distance).limit(3).all() +``` + +### Search documents within a certain distance + +Search for documents whose cosine distance from the query vector `[1, 2, 3]` is less than 0.2. + +```python +with Session(engine) as session: + distance = Document.embedding.cosine_distance([1, 2, 3]).label('distance') + results = session.query( + Document, distance + ).filter(distance < 0.2).order_by(distance).limit(3).all() +``` + +## See also + +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) +- [Vector Search Index](/tidb-cloud/vector-search-index.md) diff --git a/tidb-cloud/vector-search-integration-overview.md b/tidb-cloud/vector-search-integration-overview.md new file mode 100644 index 0000000000000..2e3be236cf743 --- /dev/null +++ b/tidb-cloud/vector-search-integration-overview.md @@ -0,0 +1,71 @@ +--- +title: Vector Search Integration Overview +summary: An overview of TiDB Vector Search integration, including supported AI frameworks, embedding models, and ORM libraries. +--- + +# Vector Search Integration Overview + +This document provides an overview of TiDB Vector Search integration, including supported AI frameworks, embedding models, and Object Relational Mapping (ORM) libraries. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## AI frameworks + +TiDB provides official support for the following AI frameworks, enabling you to easily integrate AI applications developed based on these frameworks with TiDB Vector Search. + +| AI frameworks | Tutorial | +| ------------- | ------------------------------------------------------------------------------------------------- | +| Langchain | [Integrate Vector Search with LangChain](/tidb-cloud/vector-search-integrate-with-langchain.md) | +| LlamaIndex | [Integrate Vector Search with LlamaIndex](/tidb-cloud/vector-search-integrate-with-llamaindex.md) | + +Moreover, you can also use TiDB for various purposes, such as document storage and knowledge graph storage for AI applications. + +## Embedding models and services + +TiDB Vector Search supports storing vectors of up to 16383 dimensions, which accommodates most embedding models. + +You can either use self-deployed open-source embedding models or third-party embedding APIs provided by third-party embedding providers to generate vectors. + +The following table lists some mainstream embedding service providers and the corresponding integration tutorials. + +| Embedding service providers | Tutorial | +| --------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| Jina AI | [Integrate Vector Search with Jina AI Embeddings API](/tidb-cloud/vector-search-integrate-with-jinaai-embedding.md) | + +## Object Relational Mapping (ORM) libraries + +You can integrate TiDB Vector Search with your ORM library to interact with the TiDB database. + +The following table lists the supported ORM libraries and the corresponding integration tutorials: + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    LanguageORM/ClientHow to installTutorial
    PythonTiDB Vector Clientpip install tidb-vector[client]Get Started with Vector Search Using Python
    SQLAlchemypip install tidb-vectorIntegrate TiDB Vector Search with SQLAlchemy
    peeweepip install tidb-vectorIntegrate TiDB Vector Search with peewee
    Djangopip install django-tidb[vector]Integrate TiDB Vector Search with Django
    diff --git a/tidb-cloud/vector-search-limitations.md b/tidb-cloud/vector-search-limitations.md new file mode 100644 index 0000000000000..a3b72c488cd77 --- /dev/null +++ b/tidb-cloud/vector-search-limitations.md @@ -0,0 +1,42 @@ +--- +title: Vector Search Limitations +summary: Learn the limitations of the TiDB Vector Search. +--- + +# Vector Search Limitations + +This document describes the known limitations of TiDB Vector Search. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## Vector data type limitations + +- Each [vector](/tidb-cloud/vector-search-data-types.md) supports up to 16383 dimensions. +- Vector data types cannot store `NaN`, `Infinity`, or `-Infinity` values. +- Vector data types cannot store double-precision floating-point numbers. If you insert or store double-precision floating-point numbers in vector columns, TiDB converts them to single-precision floating-point numbers. +- Vector columns cannot be used in primary keys, unique indexes, or partition keys. To accelerate the vector search performance, use [Vector Search Index](/tidb-cloud/vector-search-index.md). +- A table can have multiple vector columns. However, there is [a limit on the total number of columns in a table](/tidb-limitations.md#limitations-on-a-single-table). +- Currently, TiDB does not support dropping a vector column with a vector index. To drop such a column, drop the vector index first, then drop the vector column. +- Currently, TiDB does not support modifying a vector column to other data types such as `JSON` and `VARCHAR`. + +## Vector index limitations + +- Vector index is used for vector search. It cannot accelerate other queries like range queries or equality queries. Thus, it is not possible to create a vector index on a non-vector column, or on multiple vector columns. +- A table can have multiple vector indexes. However, there is [a limit on the total number of indexes in a table](/tidb-limitations.md#limitations-on-a-single-table). +- Creating multiple vector indexes on the same column is allowed only if they use different distance functions. +- Currently, only `VEC_COSINE_DISTANCE()` and `VEC_L2_DISTANCE()` are supported as the distance functions for vector indexes. +- Currently, TiDB does not support dropping a vector column with a vector index. To drop such a column, drop the vector index first, then drop the vector column. +- Currently, TiDB does not support setting vector index as [invisible](/sql-statements/sql-statement-alter-index.md). + +## Compatibility with TiDB tools + +- The Data Migration feature in the TiDB Cloud console does not support migrating or replicating MySQL 9.0 vector data types to TiDB Cloud. + +## Feedback + +We value your feedback and are always here to help: + +- [Join our Discord](https://discord.gg/zcqexutz2R) +- [Visit our Support Portal](https://tidb.support.pingcap.com/) diff --git a/tidb-cloud/vector-search-overview.md b/tidb-cloud/vector-search-overview.md new file mode 100644 index 0000000000000..612022eae176a --- /dev/null +++ b/tidb-cloud/vector-search-overview.md @@ -0,0 +1,72 @@ +--- +title: Vector Search (Beta) Overview +summary: Learn about Vector Search in TiDB. This feature provides an advanced search solution for performing semantic similarity searches across various data types, including documents, images, audio, and video. +--- + +# Vector Search (Beta) Overview + +TiDB Vector Search (beta) provides an advanced search solution for performing semantic similarity searches across various data types, including documents, images, audio, and video. This feature enables developers to easily build scalable applications with generative artificial intelligence (AI) capabilities using familiar MySQL skills. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## Concepts + +Vector search is a search method that prioritizes the meaning of your data to deliver relevant results. + +Unlike traditional full-text search, which relies on exact keyword matching and word frequency, vector search converts various data types (such as text, images, or audio) into high-dimensional vectors and queries based on the similarity between these vectors. This search method captures the semantic meaning and contextual information of the data, leading to a more precise understanding of user intent. + +Even when the search terms do not exactly match the content in the database, vector search can still provide results that align with the user's intent by analyzing the semantics of the data. + +For example, a full-text search for "a swimming animal" only returns results containing these exact keywords. In contrast, vector search can return results for other swimming animals, such as fish or ducks, even if these results do not contain the exact keywords. + +### Vector embedding + +A vector embedding, also known as an embedding, is a sequence of numbers that represents real-world objects in a high-dimensional space. It captures the meaning and context of unstructured data, such as documents, images, audio, and videos. + +Vector embeddings are essential in machine learning and serve as the foundation for semantic similarity searches. + +TiDB introduces [Vector data types](/tidb-cloud/vector-search-data-types.md) and [Vector search index](/tidb-cloud/vector-search-index.md) designed to optimize the storage and retrieval of vector embeddings, enhancing their use in AI applications. You can store vector embeddings in TiDB and perform vector search queries to find the most relevant data using these data types. + +### Embedding model + +Embedding models are algorithms that transform data into [vector embeddings](#vector-embedding). + +Choosing an appropriate embedding model is crucial for ensuring the accuracy and relevance of semantic search results. For unstructured text data, you can find top-performing text embedding models on the [Massive Text Embedding Benchmark (MTEB) Leaderboard](https://huggingface.co/spaces/mteb/leaderboard). + +To learn how to generate vector embeddings for your specific data types, refer to integration tutorials or examples of embedding models. + +## How vector search works + +After converting raw data into vector embeddings and storing them in TiDB, your application can execute vector search queries to find the data most semantically or contextually relevant to a user's query. + +TiDB Vector Search identifies the top-k nearest neighbor (KNN) vectors by using a [distance function](/tidb-cloud/vector-search-functions-and-operators.md) to calculate the distance between the given vector and vectors stored in the database. The vectors closest to the given vector in the query represent the most similar data in meaning. + +![The Schematic TiDB Vector Search](/media/vector-search/embedding-search.png) + +As a relational database with integrated vector search capabilities, TiDB enables you to store data and their corresponding vector representations (that is, vector embeddings) together in one database. You can choose any of the following ways for storage: + +- Store data and their corresponding vector representations in different columns of the same table. +- Store data and their corresponding vector representation in different tables. In this way, you need to use `JOIN` queries to combine the tables when retrieving data. + +## Use cases + +### Retrieval-Augmented Generation (RAG) + +Retrieval-Augmented Generation (RAG) is an architecture designed to optimize the output of Large Language Models (LLMs). By using vector search, RAG applications can store vector embeddings in the database and retrieve relevant documents as additional context when the LLM generates responses, thereby improving the quality and relevance of the answers. + +### Semantic search + +Semantic search is a search technology that returns results based on the meaning of a query, rather than simply matching keywords. It interprets the meaning across different languages and various types of data (such as text, images, and audio) using embeddings. Vector search algorithms then use these embeddings to find the most relevant data that satisfies the user's query. + +### Recommendation engine + +A recommendation engine is a system that proactively suggests content, products, or services that are relevant and personalized to users. It accomplishes this by creating embeddings that represent user behavior and preferences. These embeddings help the system identify similar items that other users have interacted with or shown interest in. This increases the likelihood that the recommendations will be both relevant and appealing to the user. + +## See also + +To get started with TiDB Vector Search, see the following documents: + +- [Get started with vector search using Python](/tidb-cloud/vector-search-get-started-using-python.md) +- [Get started with vector search using SQL](/tidb-cloud/vector-search-get-started-using-sql.md) diff --git a/tidb-configuration-file.md b/tidb-configuration-file.md index b3ba8ee1b4ab4..ce6c6809211cb 100644 --- a/tidb-configuration-file.md +++ b/tidb-configuration-file.md @@ -1,7 +1,6 @@ --- title: TiDB Configuration File summary: Learn the TiDB configuration file options that are not involved in command line options. -aliases: ['/docs/dev/tidb-configuration-file/','/docs/dev/reference/configuration/tidb-server/configuration-file/'] --- @@ -9,7 +8,7 @@ aliases: ['/docs/dev/tidb-configuration-file/','/docs/dev/reference/configuratio # TiDB Configuration File -The TiDB configuration file supports more options than command-line parameters. You can download the default configuration file [`config.toml.example`](https://github.com/pingcap/tidb/blob/master/pkg/config/config.toml.example) and rename it to `config.toml`. This document describes only the options that are not involved in [command line options](/command-line-flags-for-tidb-configuration.md). +The TiDB configuration file supports more options than command-line parameters. You can download the default configuration file [`config.toml.example`](https://github.com/pingcap/tidb/blob/release-7.5/pkg/config/config.toml.example) and rename it to `config.toml`. This document describes only the options that are not involved in [command line options](/command-line-flags-for-tidb-configuration.md). > **Tip:** > @@ -92,7 +91,7 @@ The TiDB configuration file supports more options than command-line parameters. + When [`enable-global-kill`](#enable-global-kill-new-in-v610) is `false`, `compatible-kill-query` controls whether you need to append the `TIDB` keyword when killing a query. - When `compatible-kill-query` is `false`, the behavior of `KILL xxx` in TiDB is different from that in MySQL. To kill a query in TiDB, you need to append the `TIDB` keyword, such as `KILL TIDB xxx`. - When `compatible-kill-query` is `true`, to kill a query in TiDB, there is no need to append the `TIDB` keyword. It is **STRONGLY NOT RECOMMENDED** to set `compatible-kill-query` to `true` in your configuration file UNLESS you are certain that clients will be always connected to the same TiDB instance. This is because pressing Control+C in the default MySQL client opens a new connection in which `KILL` is executed. If there is a proxy between the client and the TiDB cluster, the new connection might be routed to a different TiDB instance, which possibly kills a different session by mistake. -+ When [`enable-global-kill`](#enable-global-kill-new-in-v610) is `true`, `KILL xxx` and `KILL TIDB xxx` have the same effect, but using Control+C to kill a query is not supported. ++ When [`enable-global-kill`](#enable-global-kill-new-in-v610) is `true`, `KILL xxx` and `KILL TIDB xxx` have the same effect. + For more information about the `KILL` statement, see [KILL [TIDB]](/sql-statements/sql-statement-kill.md). ### `check-mb4-value-in-utf8` @@ -158,7 +157,13 @@ The TiDB configuration file supports more options than command-line parameters. - Sets the maximum allowable length of the newly created index. - Default value: `3072` - Unit: byte -- Currently, the valid value range is `[3072, 3072*4]`. MySQL and TiDB (version < v3.0.11) do not have this configuration item, but both limit the length of the newly created index. This limit in MySQL is `3072`. In TiDB (version =< 3.0.7), this limit is `3072*4`. In TiDB (3.0.7 < version < 3.0.11), this limit is `3072`. This configuration is added to be compatible with MySQL and earlier versions of TiDB. +- Range: `[3072, 3072*4]` +- Compatibility: + - MySQL: the maximum index length is fixed at 3072 bytes. + - Earlier versions of TiDB: + - v3.0.7 and earlier: the maximum index length is fixed at 3072 × 4 bytes. + - v3.0.8 ~ v3.0.10: the maximum index length is fixed at 3072 bytes. + - v3.0.11 and later versions: TiDB introduces the `max-index-length` configuration item to ensure compatibility with different TiDB versions and with MySQL. ### `table-column-count-limit` New in v5.0 @@ -253,7 +258,7 @@ The TiDB configuration file supports more options than command-line parameters. > - In TiDB, the `zone` label is specially used to specify the zone where a server is located. If `zone` is set to a non-null value, the corresponding value is automatically used by features such as [`txn-score`](/system-variables.md#txn_scope) and [`Follower read`](/follower-read.md). > - The `group` label has a special use in TiDB Operator. For clusters deployed using [TiDB Operator](/tidb-operator-overview.md), it is **NOT** recommended that you specify the `group` label manually. -## Log +## log Configuration items related to log. @@ -300,7 +305,7 @@ Configuration items related to log. - Outputs the threshold value of consumed time in the slow log. - Default value: `300` - Unit: Milliseconds -- If the value in a query is larger than the default value, it is a slow query and is output to the slow log. +- When the time consumed by a query is larger than this value, this query is considered as a slow query and its log is output to the slow query log. Note that when the output level of [`log.level`](#level) is `"debug"`, all queries are recorded in the slow query log, regardless of the setting of this parameter. - Since v6.1.0, the threshold value of consumed time in the slow log is specified by the TiDB configuration item [`instance.tidb_slow_log_threshold`](/tidb-configuration-file.md#tidb_slow_log_threshold) or the system variable [`tidb_slow_log_threshold`](/system-variables.md#tidb_slow_log_threshold). `slow-threshold` still takes effect. But if `slow-threshold` and `instance.tidb_slow_log_threshold` are set at the same time, the latter takes effect. ### `record-plan-in-slow-log` @@ -355,7 +360,7 @@ Configuration items related to log files. - Default value: `0` - All the log files are retained by default. If you set it to `7`, seven log files are retained at maximum. -## Security +## security Configuration items related to security. @@ -400,6 +405,11 @@ Configuration items related to security. - The path of the SSL private key file used to connect TiKV or PD with TLS. - Default value: "" +### `cluster-verify-cn` + +- A list of acceptable X.509 Common Names in certificates presented by clients. Requests are permitted only when the presented Common Name is an exact match with one of the entries in the list. +- Default value: [], which means that the client certificate CN check is disabled. + ### `spilled-file-encryption-method` + Determines the encryption method used for saving the spilled files to disk. @@ -419,20 +429,12 @@ Configuration items related to security. ### `auth-token-jwks` New in v6.4.0 -> **Warning:** -> -> The `tidb_auth_token` authentication method is used only for the internal operation of TiDB Cloud. **DO NOT** change the value of this configuration. - -- Set the local file path of the JSON Web Key Sets (JWKS) for the `tidb_auth_token` authentication method. +- Set the local file path of the JSON Web Key Sets (JWKS) for the [`tidb_auth_token`](/security-compatibility-with-mysql.md#tidb_auth_token) authentication method. - Default value: `""` ### `auth-token-refresh-interval` New in v6.4.0 -> **Warning:** -> -> The `tidb_auth_token` authentication method is used only for the internal operation of TiDB Cloud. **DO NOT** change the value of this configuration. - -- Set the JWKS refresh interval for the `tidb_auth_token` authentication method. +- Set the JWKS refresh interval for the [`tidb_auth_token`](/security-compatibility-with-mysql.md#tidb_auth_token) authentication method. - Default value: `1h` ### `disconnect-on-expired-password` New in v6.5.0 @@ -444,21 +446,17 @@ Configuration items related to security. ### `session-token-signing-cert` New in v6.4.0 -> **Warning:** -> -> The feature controlled by this parameter is under development. **Do not modify the default value**. - ++ The certificate file path, which is used by [TiProxy](https://docs.pingcap.com/tidb/stable/tiproxy-overview) for session migration. + Default value: "" ++ Empty value will cause TiProxy session migration to fail. To enable session migration, all TiDB nodes must set this to the same certificate and key. This means that you should store the same certificate and key on every TiDB node. ### `session-token-signing-key` New in v6.4.0 -> **Warning:** -> -> The feature controlled by this parameter is under development. **Do not modify the default value**. - ++ The key file path used by [TiProxy](https://docs.pingcap.com/tidb/stable/tiproxy-overview) for session migration. + Default value: "" ++ Refer to the descriptions of [`session-token-signing-cert`](#session-token-signing-cert-new-in-v640). -## Performance +## performance Configuration items related to performance. @@ -490,7 +488,7 @@ Configuration items related to performance. - Default value: `5000` - If a transaction does not roll back or commit after the number of statements exceeds `stmt-count-limit`, TiDB returns the `statement count 5001 exceeds the transaction limitation, autocommit = false` error. This configuration takes effect **only** in the retryable optimistic transaction. If you use the pessimistic transaction or have disabled the transaction retry, the number of statements in a transaction is not limited by this configuration. -### `txn-entry-size-limit` New in v5.0 +### `txn-entry-size-limit` New in v4.0.10 and v5.0.0 - The size limit of a single row of data in TiDB. - Default value: `6291456` (in bytes) @@ -506,7 +504,7 @@ Configuration items related to performance. - In a single transaction, the total size of key-value records cannot exceed this value. The maximum value of this parameter is `1099511627776` (1 TB). Note that if you have used the binlog to serve the downstream consumer Kafka (such as the `arbiter` cluster), the value of this parameter must be no more than `1073741824` (1 GB). This is because 1 GB is the upper limit of a single message size that Kafka can process. Otherwise, an error is returned if this limit is exceeded. - In TiDB v6.5.0 and later versions, this configuration is no longer recommended. The memory size of a transaction will be accumulated into the memory usage of the session, and the [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query) variable will take effect when the session memory threshold is exceeded. To be compatible with previous versions, this configuration works as follows when you upgrade from an earlier version to TiDB v6.5.0 or later: - If this configuration is not set or is set to the default value (`104857600`), after an upgrade, the memory size of a transaction will be accumulated into the memory usage of the session, and the `tidb_mem_quota_query` variable will take effect. - - If this configuration is not defaulted (`104857600`), it still takes effect and its behavior on controling the size of a single transaction remains unchanged before and after the upgrade. This means that the memory size of the transaction is not controlled by the `tidb_mem_quota_query` variable. + - If this configuration is not defaulted (`104857600`), it still takes effect and its behavior on controlling the size of a single transaction remains unchanged before and after the upgrade. This means that the memory size of the transaction is not controlled by the `tidb_mem_quota_query` variable. ### `tcp-keep-alive` @@ -523,6 +521,10 @@ Configuration items related to performance. - Default value: `true` - TiDB supports executing the `JOIN` statement without any condition (the `WHERE` field) of both sides tables by default; if you set the value to `false`, the server refuses to execute when such a `JOIN` statement appears. +> **Note:** +> +> When creating a cluster, **DO NOT** set `cross-join` to false. Otherwise, the cluster will fail to start up. + ### `stats-lease` - The time interval of reloading statistics, updating the number of table rows, checking whether it is needed to perform the automatic analysis, using feedback to update statistics and loading statistics of columns. @@ -551,6 +553,10 @@ Configuration items related to performance. - Value options: The default value `NO_PRIORITY` means that the priority for statements is not forced to change. Other options are `LOW_PRIORITY`, `DELAYED`, and `HIGH_PRIORITY` in ascending order. - Since v6.1.0, the priority for all statements is determined by the TiDB configuration item [`instance.tidb_force_priority`](/tidb-configuration-file.md#tidb_force_priority) or the system variable [`tidb_force_priority`](/system-variables.md#tidb_force_priority). `force-priority` still takes effect. But if `force-priority` and `instance.tidb_force_priority` are set at the same time, the latter takes effect. +> **Note:** +> +> Starting from v6.6.0, TiDB supports [Resource Control](/tidb-resource-control.md). You can use this feature to execute SQL statements with different priorities in different resource groups. By configuring proper quotas and priorities for these resource groups, you can gain better scheduling control for SQL statements with different priorities. When resource control is enabled, statement priority will no longer take effect. It is recommended that you use [Resource Control](/tidb-resource-control.md) to manage resource usage for different SQL statements. + ### `distinct-agg-push-down` - Determines whether the optimizer executes the operation that pushes down the aggregation function with `Distinct` (such as `select count(distinct a) from t`) to Coprocessors. @@ -580,6 +586,11 @@ Configuration items related to performance. + Default value: `1000` + Currently, the valid value range is `[1, 100000]`. +### `concurrently-init-stats` New in v8.1.0 and v7.5.2 + ++ Controls whether to initialize statistics concurrently during TiDB startup. ++ Default value: `false` + ### `lite-init-stats` New in v7.1.0 + Controls whether to use lightweight statistics initialization during TiDB startup. @@ -587,7 +598,7 @@ Configuration items related to performance. + When the value of `lite-init-stats` is `true`, statistics initialization does not load any histogram, TopN, or Count-Min Sketch of indexes or columns into memory. When the value of `lite-init-stats` is `false`, statistics initialization loads histograms, TopN, and Count-Min Sketch of indexes and primary keys into memory but does not load any histogram, TopN, or Count-Min Sketch of non-primary key columns into memory. When the optimizer needs the histogram, TopN, and Count-Min Sketch of a specific index or column, the necessary statistics are loaded into memory synchronously or asynchronously (controlled by [`tidb_stats_load_sync_wait`](/system-variables.md#tidb_stats_load_sync_wait-new-in-v540)). + Setting `lite-init-stats` to `true` speeds up statistics initialization and reduces TiDB memory usage by avoiding unnecessary statistics loading. For details, see [Load statistics](/statistics.md#load-statistics). -### `force-init-stats` New in v7.1.0 +### `force-init-stats` New in v6.5.7 and v7.1.0 + Controls whether to wait for statistics initialization to finish before providing services during TiDB startup. + Default value: `false` for versions earlier than v7.2.0, `true` for v7.2.0 and later versions. @@ -687,10 +698,14 @@ Configuration items related to opentracing.reporter. ### `grpc-compression-type` -- Specifies the compression type used for data transfer between TiDB and TiKV nodes. The default value is `"none"`, which means no compression. To enable the gzip compression, set this value to `"gzip"`. +- Specifies the compression type used for data transfer from TiDB nodes to TiKV nodes. The default value is `"none"`, which means no compression. To enable the gzip compression, set this value to `"gzip"`. - Default value: `"none"` - Value options: `"none"`, `"gzip"` +> **Note:** +> +> The compression algorithm for response messages returned from TiKV nodes to TiDB nodes is controlled by the TiKV configuration item [`grpc-compression-type`](/tikv-configuration-file.md#grpc-compression-type). + ### `commit-timeout` - The maximum timeout when executing a transaction commit. @@ -716,9 +731,19 @@ Configuration items related to opentracing.reporter. ### `overload-threshold` -- The threshold of the TiKV load. If the TiKV load exceeds this threshold, more `batch` packets are collected to relieve the pressure of TiKV. It is valid only when the value of `tikv-client.max-batch-size` is greater than `0`. It is recommended not to modify this value. +- The threshold of the TiKV load. If the TiKV load exceeds this threshold, more `batch` packets are collected to relieve the pressure of TiKV. This configuration item takes effect only when both [`tikv-client.max-batch-size`](#max-batch-size) and [`tikv-client.max-batch-wait-time`](#max-batch-wait-time) are set to values greater than `0`. It is recommended not to modify this value. - Default value: `200` +### `copr-req-timeout` New in v7.5.0 + +> **Warning:** +> +> This configuration might be deprecated in future versions. **DO NOT** change the value of this configuration. + ++ The timeout of a single Coprocessor request. ++ Default value: `60` ++ Unit: second + ## tikv-client.copr-cache New in v4.0.0 This section introduces configuration items related to the Coprocessor Cache feature. @@ -732,7 +757,7 @@ This section introduces configuration items related to the Coprocessor Cache fea ## txn-local-latches -Configuration related to the transaction latch. It is recommended to enable it when many local transaction conflicts occur. +Configuration items related to the transaction latch. These configuration items might be deprecated in the future. It is not recommended to use them. ### `enabled` @@ -789,6 +814,12 @@ Configuration related to the status of TiDB service. - Determines whether to transmit the database-related QPS metrics to Prometheus. - Default value: `false` +### `record-db-label` + +- Determines whether to transmit the database-related QPS metrics to Prometheus. +- Supports more metircs types than `record-db-qps`, for example, duration and statements. +- Default value: `false` + ## pessimistic-txn For pessimistic transaction usage, refer to [TiDB Pessimistic Transaction Mode](/pessimistic-transaction.md). @@ -849,10 +880,11 @@ Configuration items related to read isolation. ### `tidb_slow_log_threshold` -- This configuration is used to output the threshold value of the time consumed by the slow log. When the time consumed by a query is larger than this value, this query is considered as a slow log and its log is output to the slow query log. +- Outputs the threshold value of the time consumed by the slow log. - Default value: `300` - Range: `[-1, 9223372036854775807]` - Unit: Milliseconds +- When the time consumed by a query is larger than this value, this query is considered as a slow query and its log is output to the slow query log. Note that when the output level of [`log.level`](#level) is `"debug"`, all queries are recorded in the slow query log, regardless of the setting of this parameter. - Before v6.1.0, this configuration is set by `slow-threshold`. ### `in-mem-slow-query-topn-num` New in v7.3.0 @@ -890,6 +922,10 @@ Configuration items related to read isolation. - The default value `NO_PRIORITY` means that the priority for statements is not forced to change. Other options are `LOW_PRIORITY`, `DELAYED`, and `HIGH_PRIORITY` in ascending order. - Before v6.1.0, this configuration is set by `force-priority`. +> **Note:** +> +> Starting from v6.6.0, TiDB supports [Resource Control](/tidb-resource-control.md). You can use this feature to execute SQL statements with different priorities in different resource groups. By configuring proper quotas and priorities for these resource groups, you can gain better scheduling control for SQL statements with different priorities. When resource control is enabled, statement priority will no longer take effect. It is recommended that you use [Resource Control](/tidb-resource-control.md) to manage resource usage for different SQL statements. + ### `max_connections` - The maximum number of connections permitted for a single TiDB instance. It can be used for resources control. diff --git a/tidb-control.md b/tidb-control.md index 2c79e6ff347f4..e2875d63ca8ee 100644 --- a/tidb-control.md +++ b/tidb-control.md @@ -1,7 +1,6 @@ --- title: TiDB Control User Guide summary: Use TiDB Control to obtain TiDB status information for debugging. -aliases: ['/docs/dev/tidb-control/','/docs/dev/reference/tools/tidb-control/'] --- # TiDB Control User Guide diff --git a/tidb-distributed-execution-framework.md b/tidb-distributed-execution-framework.md index a2120cc0670be..d4fdcf7a0bcb7 100644 --- a/tidb-distributed-execution-framework.md +++ b/tidb-distributed-execution-framework.md @@ -1,51 +1,33 @@ --- -title: TiDB Backend Task Distributed Execution Framework -summary: Learn the use cases, limitations, usage, and implementation principles of the TiDB backend task distributed execution framework. +title: TiDB Distributed eXecution Framework (DXF) +summary: Learn the use cases, limitations, usage, and implementation principles of the TiDB Distributed eXecution Framework (DXF). --- -# TiDB Backend Task Distributed Execution Framework - - +# TiDB Distributed eXecution Framework (DXF) > **Note:** > -> Currently, this feature is only applicable to TiDB Dedicated clusters. You cannot use it on TiDB Serverless clusters. - - - -TiDB adopts a computing-storage separation architecture with excellent scalability and elasticity. Starting from v7.1.0, TiDB introduces a backend task distributed execution framework to further leverage the resource advantages of the distributed architecture. The goal of this framework is to implement unified scheduling and distributed execution of all backend tasks, and to provide unified resource management capabilities for both overall and individual backend tasks, which better meets users' expectations for resource usage. - -This document describes the use cases, limitations, usage, and implementation principles of the TiDB backend task distributed execution framework. - -> **Note:** -> -> This framework does not support the distributed execution of SQL queries. - -## Use cases and limitations - - - -In a database management system, in addition to the core transactional processing (TP) and analytical processing (AP) workloads, there are other important tasks, such as DDL operations, IMPORT INTO, TTL, Analyze, and Backup/Restore, which are called **backend tasks**. These backend tasks need to process a large amount of data in database objects (tables), so they typically have the following characteristics: +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. - +TiDB adopts a computing-storage separation architecture with excellent scalability and elasticity. Starting from v7.1.0, TiDB introduces a Distributed eXecution Framework (DXF) to further leverage the resource advantages of the distributed architecture. The goal of the DXF is to implement unified scheduling and distributed execution of tasks, and to provide unified resource management capabilities for both overall and individual tasks, which better meets users' expectations for resource usage. - +This document describes the use cases, limitations, usage, and implementation principles of the DXF. -In a database management system, in addition to the core transactional processing (TP) and analytical processing (AP) workloads, there are other important tasks, such as DDL operations, TTL, Analyze, and Backup/Restore, which are called **backend tasks**. These backend tasks need to process a large amount of data in database objects (tables), so they typically have the following characteristics: +## Use cases - +In a database management system, in addition to the core transactional processing (TP) and analytical processing (AP) workloads, there are other important tasks, such as DDL operations, IMPORT INTO, TTL, Analyze, and Backup/Restore. These tasks need to process a large amount of data in database objects (tables), so they typically have the following characteristics: - Need to process all data in a schema or a database object (table). - Might need to be executed periodically, but at a low frequency. - If the resources are not properly controlled, they are prone to affect TP and AP tasks, lowering the database service quality. -Enabling the TiDB backend task distributed execution framework can solve the above problems and has the following three advantages: +Enabling the DXF can solve the above problems and has the following three advantages: - The framework provides unified capabilities for high scalability, high availability, and high performance. -- The framework supports distributed execution of backend tasks, which can flexibly schedule the available computing resources of the entire TiDB cluster, thereby better utilizing the computing resources in a TiDB cluster. -- The framework provides unified resource usage and management capabilities for both overall and individual backend tasks. +- The DXF supports distributed execution of tasks, which can flexibly schedule the available computing resources of the entire TiDB cluster, thereby better utilizing the computing resources in a TiDB cluster. +- The DXF provides unified resource usage and management capabilities for both overall and individual tasks. -Currently, for TiDB Self-Hosted, the TiDB backend task distributed execution framework supports the distributed execution of the `ADD INDEX` and `IMPORT INTO` statements. For TiDB Cloud, the `IMPORT INTO` statement is not applicable. +Currently, the DXF supports the distributed execution of the `ADD INDEX` and `IMPORT INTO` statements. - `ADD INDEX` is a DDL statement used to create indexes. For example: @@ -54,11 +36,15 @@ Currently, for TiDB Self-Hosted, the TiDB backend task distributed execution fra CREATE INDEX idx1 ON table t1(c1); ``` -- `IMPORT INTO` is used to import data in formats such as `CSV`, `SQL`, and `PARQUET` into an empty table. For more information, see [`IMPORT INTO`](https://docs.pingcap.com/tidb/v7.2/sql-statement-import-into). +- `IMPORT INTO` is used to import data in formats such as `CSV`, `SQL`, and `PARQUET` into an empty table. For more information, see [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md). + +## Limitation + +The DXF can only schedule the distributed execution of one `ADD INDEX` task at a time. If a new `ADD INDEX` task is submitted before the current `ADD INDEX` distributed task has finished, the new task is executed through a transaction. ## Prerequisites -Before using the distributed framework, you need to enable the [Fast Online DDL](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) mode. +Before using the DXF to execute `ADD INDEX` tasks, you need to enable the [Fast Online DDL](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) mode. @@ -73,7 +59,7 @@ Before using the distributed framework, you need to enable the [Fast Online DDL] > **Note:** > -> Before you upgrade TiDB to v6.5.0 or later, it is recommended that you check whether the [`temp-dir`](/tidb-configuration-file.md#temp-dir-new-in-v630) path of TiDB is correctly mounted to an SSD disk. Make sure that the operating system user that runs TiDB has the read and write permissions for this directory. Otherwise, The DDL operations might experience unpredictable issues. This path is a TiDB configuration item, which takes effect after TiDB is restarted. Therefore, setting this configuration item in advance before upgrading can avoid another restart. +> It is recommended that you prepare at least 100 GiB of free space for the TiDB `temp-dir` directory. @@ -88,51 +74,38 @@ Adjust the following system variables related to Fast Online DDL: ## Usage -1. To enable the distributed framework, set the value of [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) to `ON`: +1. To enable the DXF, set the value of [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) to `ON`: ```sql SET GLOBAL tidb_enable_dist_task = ON; ``` - - - When background tasks are running, the statements supported by the framework (such as [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) and [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md)) are executed in a distributed manner. All TiDB nodes run background tasks by default. - - + When the DXF tasks are running, the statements supported by the framework (such as [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) and [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md)) are executed in a distributed manner. All TiDB nodes run DXF tasks by default. - - - When background tasks are running, the statements supported by the framework (such as [`ADD INDEX`](/sql-statements/sql-statement-add-index.md)) are executed in a distributed manner. All TiDB nodes run background tasks by default. - - - -2. Adjust the following system variables that might affect the distributed execution of DDL tasks according to your needs: +2. In general, for the following system variables that might affect the distributed execution of DDL tasks, it is recommended that you use their default values: * [`tidb_ddl_reorg_worker_cnt`](/system-variables.md#tidb_ddl_reorg_worker_cnt): use the default value `4`. The recommended maximum value is `16`. * [`tidb_ddl_reorg_priority`](/system-variables.md#tidb_ddl_reorg_priority) * [`tidb_ddl_error_count_limit`](/system-variables.md#tidb_ddl_error_count_limit) * [`tidb_ddl_reorg_batch_size`](/system-variables.md#tidb_ddl_reorg_batch_size): use the default value. The recommended maximum value is `1024`. -3. Starting from v7.4.0, you can adjust the number of TiDB nodes that perform background tasks according to actual needs. After deploying a TiDB cluster, you can set the instance-level system variable [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) for each TiDB node in the cluster. When `tidb_service_scope` of a TiDB node is set to `background`, the TiDB node can execute background tasks. When `tidb_service_scope` of a TiDB node is set to the default value "", the TiDB node cannot execute background tasks. If `tidb_service_scope` is not set for any TiDB node in a cluster, the TiDB distributed execution framework schedules all TiDB nodes to execute background tasks by default. +3. Starting from v7.4.0, for TiDB Self-Managed, you can adjust the number of TiDB nodes that perform the DXF tasks according to actual needs. After deploying a TiDB cluster, you can set the instance-level system variable [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) for each TiDB node in the cluster. When `tidb_service_scope` of a TiDB node is set to `background`, the TiDB node can execute the DXF tasks. When `tidb_service_scope` of a TiDB node is set to the default value "", the TiDB node cannot execute the DXF tasks. If `tidb_service_scope` is not set for any TiDB node in a cluster, the DXF schedules all TiDB nodes to execute tasks by default. + > **Note:** > - > - During the execution of a distributed task, if some TiDB nodes are offline, the distributed task randomly assigns subtasks to available TiDB nodes instead of dynamically assigning subtasks according to `tidb_service_scope`. + > - In a cluster with several TiDB nodes, it is strongly recommended to set `tidb_service_scope` to `background` on two or more TiDB nodes. If `tidb_service_scope` is set on a single TiDB node only, when the node is restarted or fails, the task will be rescheduled to other TiDB nodes that lack the `background` setting, which will affect these TiDB nodes. > - During the execution of a distributed task, changes to the `tidb_service_scope` configuration will not take effect for the current task, but will take effect from the next task. -> **Tip:** -> -> For distributed execution of `ADD INDEX` statements, you only need to set `tidb_ddl_reorg_worker_cnt`. - ## Implementation principles -The architecture of the TiDB backend task distributed execution framework is as follows: +The architecture of the DXF is as follows: -![Architecture of the TiDB backend task distributed execution framework](/media/dist-task/dist-task-architect.jpg) +![Architecture of the DXF](/media/dist-task/dist-task-architect.jpg) -As shown in the preceding diagram, the execution of backend tasks in the distributed framework is mainly handled by the following modules: +As shown in the preceding diagram, the execution of tasks in the DXF is mainly handled by the following modules: - Dispatcher: generates the distributed execution plan for each task, manages the execution process, converts the task status, and collects and feeds back the runtime task information. -- Scheduler: replicates the execution of distributed tasks among TiDB nodes to improve the efficiency of backend task execution. +- Scheduler: replicates the execution of distributed tasks among TiDB nodes to improve the efficiency of task execution. - Subtask Executor: the actual executor of distributed subtasks. In addition, the Subtask Executor returns the execution status of subtasks to the Scheduler, and the Scheduler updates the execution status of subtasks in a unified manner. - Resource pool: provides the basis for quantifying resource usage and management by pooling computing resources of the above modules. @@ -140,7 +113,7 @@ As shown in the preceding diagram, the execution of backend tasks in the distrib -* [Execution Principles and Best Practices of DDL Statements](/ddl-introduction.md) +* [Execution Principles and Best Practices of DDL Statements](/best-practices/ddl-introduction.md) diff --git a/tidb-global-sort.md b/tidb-global-sort.md index b5f3e66d7e851..346bd12bea254 100644 --- a/tidb-global-sort.md +++ b/tidb-global-sort.md @@ -12,17 +12,13 @@ summary: Learn the use cases, limitations, usage, and implementation principles > > This feature is experimental. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. - - > **Note:** > -> Currently, this feature is only applicable to TiDB Dedicated clusters. You cannot use it on TiDB Serverless clusters. - - +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. ## Overview -The TiDB Global Sort feature enhances the stability and efficiency of data import and DDL (Data Definition Language) operations. It serves as a general operator in the [distributed execution framework](/tidb-distributed-execution-framework.md), providing a global sort service on cloud. +The TiDB Global Sort feature enhances the stability and efficiency of data import and DDL (Data Definition Language) operations. It serves as a general operator in the [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md), providing a global sort service on cloud. The Global Sort feature currently only supports using Amazon S3 as cloud storage. In future releases, it will be extended to support multiple shared storage interfaces, such as POSIX, enabling seamless integration with different storage systems. This flexibility enables efficient and adaptable data sorting for various use cases. @@ -30,7 +26,7 @@ The Global Sort feature currently only supports using Amazon S3 as cloud storage The Global Sort feature enhances the stability and efficiency of `IMPORT INTO` and `CREATE INDEX`. By globally sorting the data that are processed by the tasks, it improves the stability, controllability, and scalability of writing data to TiKV. This provides an enhanced user experience for data import and DDL tasks, as well as higher-quality services. -The Global Sort feature executes tasks within the unified distributed parallel execution framework, ensuring efficient and parallel sorting of data on a global scale. +The Global Sort feature executes tasks within the unified DXF, ensuring efficient and parallel sorting of data on a global scale. ## Limitations @@ -40,7 +36,7 @@ Currently, the Global Sort feature is not used as a component of the query execu To enable Global Sort, follow these steps: -1. Enable the distributed execution framework by setting the value of [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) to `ON`: +1. Enable the DXF by setting the value of [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) to `ON`: ```sql SET GLOBAL tidb_enable_dist_task = ON; @@ -50,17 +46,25 @@ To enable Global Sort, follow these steps: 2. Set [`tidb_cloud_storage_uri`](/system-variables.md#tidb_cloud_storage_uri-new-in-v740) to a correct cloud storage path. See [an example](/br/backup-and-restore-storages.md). + ```sql + SET GLOBAL tidb_cloud_storage_uri = 's3://my-bucket/test-data?role-arn=arn:aws:iam::888888888888:role/my-role' + ``` + 2. Set [`tidb_cloud_storage_uri`](/system-variables.md#tidb_cloud_storage_uri-new-in-v740) to a correct cloud storage path. See [an example](https://docs.pingcap.com/tidb/stable/backup-and-restore-storages). - - ```sql SET GLOBAL tidb_cloud_storage_uri = 's3://my-bucket/test-data?role-arn=arn:aws:iam::888888888888:role/my-role' ``` + + +> **Note:** +> +> For [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md), you can also specify the cloud storage path using the [`CLOUD_STORAGE_URI`](/sql-statements/sql-statement-import-into.md#withoptions) option. If both [`tidb_cloud_storage_uri`](/system-variables.md#tidb_cloud_storage_uri-new-in-v740) and `CLOUD_STORAGE_URI` are configured with a valid cloud storage path, the configuration of `CLOUD_STORAGE_URI` takes effect for [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md). + ## Implementation principles The algorithm of the Global Sort feature is as follows: diff --git a/tidb-in-kubernetes.md b/tidb-in-kubernetes.md index f85259fb9bd09..1963d64c80bcd 100644 --- a/tidb-in-kubernetes.md +++ b/tidb-in-kubernetes.md @@ -1,7 +1,6 @@ --- title: Deploy a TiDB Cluster on Kubernetes summary: Learn how to deploy a TiDB cluster on Kubernetes. -aliases: ['/docs/tidb-in-kubernetes/dev/'] --- # Deploy a TiDB Cluster on Kubernetes diff --git a/tidb-lightning/deploy-tidb-lightning.md b/tidb-lightning/deploy-tidb-lightning.md index cd678a4d04987..f2afceff11ebc 100644 --- a/tidb-lightning/deploy-tidb-lightning.md +++ b/tidb-lightning/deploy-tidb-lightning.md @@ -1,7 +1,6 @@ --- title: Deploy TiDB Lightning summary: Deploy TiDB Lightning to quickly import large amounts of new data. -aliases: ['/docs/dev/tidb-lightning/deploy-tidb-lightning/','/docs/dev/reference/tools/tidb-lightning/deployment/'] --- # Deploy TiDB Lightning @@ -40,87 +39,7 @@ tar -zxvf tidb-lightning-${version}-linux-amd64.tar.gz chmod +x tidb-lightning ``` -In this command, - -- `-B test`: means the data is exported from the `test` database. -- `-f test.t[12]`: means only the `test.t1` and `test.t2` tables are exported. -- `-t 16`: means 16 threads are used to export the data. -- `-F 256MB`: means a table is partitioned into chunks and one chunk is 256 MB. - -If the data source consists of CSV files, see [CSV support](/tidb-lightning/tidb-lightning-data-source.md#csv) for configuration. - -## Deploy TiDB Lightning - -This section describes how to [deploy TiDB Lightning manually](#deploy-tidb-lightning-manually). - -### Deploy TiDB Lightning manually - -#### Step 1: Deploy a TiDB cluster - -Before importing data, you need to have a deployed TiDB cluster. It is highly recommended to use the latest stable version. - -You can find deployment instructions in [TiDB Quick Start Guide](/quick-start-with-tidb.md). - -#### Step 2: Download the TiDB Lightning installation package - -Refer to the [Download TiDB Tools](/download-ecosystem-tools.md) document to download the TiDB Lightning package. - -> **Note:** -> -> TiDB Lightning is compatible with TiDB clusters of earlier versions. It is recommended that you download the latest stable version of the TiDB Lightning installation package. - -#### Step 3: Start `tidb-lightning` - -1. Upload `bin/tidb-lightning` and `bin/tidb-lightning-ctl` from the tool set. - -2. Mount the data source onto the same machine. - -3. Configure `tidb-lightning.toml`. For configurations that do not appear in the template below, TiDB Lightning writes a configuration error to the log file and exits. - - `sorted-kv-dir` sets the temporary storage directory for the sorted Key-Value files. The directory must be empty, and the storage space **must be greater than the size of the dataset to be imported**. See [Downstream storage space requirements](/tidb-lightning/tidb-lightning-requirements.md#storage-space-of-the-target-database) for details. - - ```toml - [lightning] - # The concurrency number of data. It is set to the number of logical CPU - # cores by default. When deploying together with other components, you can - # set it to 75% of the size of logical CPU cores to limit the CPU usage. - # region-concurrency = - - # Logging - level = "info" - file = "tidb-lightning.log" - - [tikv-importer] - # Sets the backend to the "local" mode. - backend = "local" - # Sets the directory of temporary local storage. - sorted-kv-dir = "/mnt/ssd/sorted-kv-dir" - - [mydumper] - # Local source data directory - data-source-dir = "/data/my_database" - - [tidb] - # Configuration of any TiDB server from the cluster - host = "172.16.31.1" - port = 4000 - user = "root" - password = "" - # Table schema information is fetched from TiDB via this status-port. - status-port = 10080 - # An address of pd-server. - pd-addr = "172.16.31.4:2379" - ``` - - The above only shows the essential settings. See the [Configuration](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-global) section for the full list of settings. - -4. Run `tidb-lightning`. - - ```sh - nohup ./tidb-lightning -config tidb-lightning.toml > nohup.out & - ``` - -## Upgrade TiDB Lightning +### Upgrade TiDB Lightning You can upgrade TiDB Lightning by replacing the binaries alone without further configurations. After the upgrade, you need to restart TiDB Lightning. For details, see [How to properly restart TiDB Lightning](/tidb-lightning/tidb-lightning-faq.md#how-to-properly-restart-tidb-lightning). diff --git a/tidb-lightning/monitor-tidb-lightning.md b/tidb-lightning/monitor-tidb-lightning.md index 6b922b4445c37..01b9c6e823eb8 100644 --- a/tidb-lightning/monitor-tidb-lightning.md +++ b/tidb-lightning/monitor-tidb-lightning.md @@ -1,7 +1,6 @@ --- title: TiDB Lightning Monitoring summary: Learn about the monitor configuration and monitoring metrics of TiDB Lightning. -aliases: ['/docs/dev/tidb-lightning/monitor-tidb-lightning/','/docs/dev/reference/tools/tidb-lightning/monitor/'] --- # TiDB Lightning Monitoring diff --git a/tidb-lightning/tidb-lightning-checkpoints.md b/tidb-lightning/tidb-lightning-checkpoints.md index c68bc8201496d..e874f6290efb3 100644 --- a/tidb-lightning/tidb-lightning-checkpoints.md +++ b/tidb-lightning/tidb-lightning-checkpoints.md @@ -1,7 +1,6 @@ --- title: TiDB Lightning Checkpoints summary: Use checkpoints to avoid redoing the previously completed tasks before the crash. -aliases: ['/docs/dev/tidb-lightning/tidb-lightning-checkpoints/','/docs/dev/reference/tools/tidb-lightning/checkpoints/'] --- # TiDB Lightning Checkpoints diff --git a/tidb-lightning/tidb-lightning-configuration.md b/tidb-lightning/tidb-lightning-configuration.md index 858396c01c42a..ed03491f7e829 100644 --- a/tidb-lightning/tidb-lightning-configuration.md +++ b/tidb-lightning/tidb-lightning-configuration.md @@ -1,7 +1,6 @@ --- title: TiDB Lightning Configuration summary: Learn about the CLI usage and sample configuration in TiDB Lightning. -aliases: ['/docs/dev/tidb-lightning/tidb-lightning-configuration/','/docs/dev/reference/tools/tidb-lightning/config/'] --- # TiDB Lightning Configuration @@ -374,17 +373,17 @@ index-serial-scan-concurrency = 20 checksum-table-concurrency = 2 # The default SQL mode used to parse and execute the SQL statements. -sql-mode = "ONLY_FULL_GROUP_BY,NO_ENGINE_SUBSTITUTION" +sql-mode = "ONLY_FULL_GROUP_BY,NO_AUTO_CREATE_USER" # Sets maximum packet size allowed for SQL connections. # Set this to 0 to automatically fetch the `max_allowed_packet` variable from server on every connection. max-allowed-packet = 67_108_864 # Whether to use TLS for SQL connections. Valid values are: -# * "" - force TLS (same as "cluster") if [tidb.security] section is populated, otherwise same as "false" -# * "false" - disable TLS -# * "cluster" - force TLS and verify the server's certificate with the CA specified in the [tidb.security] section -# * "skip-verify" - force TLS but do not verify the server's certificate (insecure!) -# * "preferred" - same as "skip-verify", but if the server does not support TLS, fallback to unencrypted connection +# - "": if configuration items in the [tidb.security] section are configured, TiDB Lightning requires TLS for SQL connections (same behavior as "cluster"). Otherwise, it uses an unencrypted connection. +# - "false": same behavior as "". +# - "cluster": requires TLS and verifies the server's certificate with the CA specified in the [tidb.security] section. +# - "skip-verify": requires TLS but does not verify the server's certificate (insecure). If the server does not support TLS, the connection falls back to an unencrypted state. +# - "preferred": same behavior as "skip-verify". # tls = "" # Specifies certificates and keys for TLS-enabled MySQL connections. @@ -397,6 +396,10 @@ max-allowed-packet = 67_108_864 # Private key of this service. Default to copy of `security.key-path` # key-path = "/path/to/lightning.key" +# Sets other TiDB session variables +# [tidb.session-vars] +# tidb_enable_clustered_index = "OFF" + # In the physical import mode, when data importing is complete, TiDB Lightning can # automatically perform the Checksum and Analyze operations. It is recommended # to leave these as true in the production environment. @@ -436,54 +439,3 @@ log-progress = "5m" # The default value is 60 seconds. # check-disk-quota = "60s" ``` - -## Command line parameters - -### Usage of `tidb-lightning` - -| Parameter | Explanation | Corresponding setting | -|:----|:----|:----| -| --config *file* | Reads global configuration from *file*. If not specified, the default configuration would be used. | | -| -V | Prints program version | | -| -d *directory* | Directory or [external storage URI](/external-storage-uri.md) of the data dump to read from | `mydumper.data-source-dir` | -| -L *level* | Log level: debug, info, warn, error, fatal (default = info) | `lightning.log-level` | -| -f *rule* | [Table filter rules](/table-filter.md) (can be specified multiple times) | `mydumper.filter` | -| --backend *[backend](/tidb-lightning/tidb-lightning-overview.md)* | Select an import mode. `local` refers to the physical import mode; `tidb` refers to the logical import mode. | `local` | -| --log-file *file* | Log file path. By default, it is `/tmp/lightning.log.{timestamp}`. If set to '-', it means that the log files will be output to stdout. | `lightning.log-file` | -| --status-addr *ip:port* | Listening address of the TiDB Lightning server | `lightning.status-port` | -| --pd-urls *host:port* | PD endpoint address | `tidb.pd-addr` | -| --tidb-host *host* | TiDB server host | `tidb.host` | -| --tidb-port *port* | TiDB server port (default = 4000) | `tidb.port` | -| --tidb-status *port* | TiDB status port (default = 10080) | `tidb.status-port` | -| --tidb-user *user* | User name to connect to TiDB | `tidb.user` | -| --tidb-password *password* | Password to connect to TiDB. The password can either be plaintext or Base64 encoded. | `tidb.password` | -| --enable-checkpoint *bool* | Whether to enable checkpoints (default = true) | `checkpoint.enable` | -| --analyze *level* | Analyze tables after importing. Available values are "required", "optional" (default value), and "off" | `post-restore.analyze` | -| --checksum *level* | Compare checksum after importing. Available values are "required" (default value), "optional", and "off" | `post-restore.checksum` | -| --check-requirements *bool* | Check cluster version compatibility before starting the task, and check whether TiKV has more than 10% free space left during running time. (default = true) | `lightning.check-requirements` | -| --ca *file* | CA certificate path for TLS connection | `security.ca-path` | -| --cert *file* | Certificate path for TLS connection | `security.cert-path` | -| --key *file* | Private key path for TLS connection | `security.key-path` | -| --server-mode | Start TiDB Lightning in server mode | `lightning.server-mode` | - -If a command line parameter and the corresponding setting in the configuration file are both provided, the command line parameter will be used. For example, running `./tidb-lightning -L debug --config cfg.toml` would always set the log level to "debug" regardless of the content of `cfg.toml`. - -## Usage of `tidb-lightning-ctl` - -This tool can execute various actions given one of the following parameters: - -| Parameter | Explanation | -|:----|:----| -| --compact | Performs a full compaction | -| --switch-mode *mode* | Switches every TiKV store to the given mode: normal, import | -| --fetch-mode | Prints the current mode of every TiKV store | -| --import-engine *uuid* | Imports the closed engine file from TiKV Importer into the TiKV cluster | -| --cleanup-engine *uuid* | Deletes the engine file from TiKV Importer | -| --checkpoint-dump *folder* | Dumps current checkpoint as CSVs into the folder | -| --checkpoint-error-destroy *tablename* | Removes the checkpoint and drops the table if it caused error | -| --checkpoint-error-ignore *tablename* | Ignores any error recorded in the checkpoint involving the given table | -| --checkpoint-remove *tablename* | Unconditionally removes the checkpoint of the table | - -The *tablename* must either be a qualified table name in the form `` `db`.`tbl` `` (including the backquotes), or the keyword "all". - -Additionally, all parameters of `tidb-lightning` described in the section above are valid in `tidb-lightning-ctl`. diff --git a/tidb-lightning/tidb-lightning-data-source.md b/tidb-lightning/tidb-lightning-data-source.md index df3db7ec2e33b..b5d07c08e4eb9 100644 --- a/tidb-lightning/tidb-lightning-data-source.md +++ b/tidb-lightning/tidb-lightning-data-source.md @@ -1,7 +1,6 @@ --- title: TiDB Lightning Data Sources summary: Learn all the data sources supported by TiDB Lightning. -aliases: ['/docs/dev/tidb-lightning/migrate-from-csv-using-tidb-lightning/','/docs/dev/reference/tools/tidb-lightning/csv/','/tidb/dev/migrate-from-csv-using-tidb-lightning/'] --- # TiDB Lightning Data Sources @@ -12,7 +11,7 @@ To specify the data source for TiDB Lightning, use the following configuration: ```toml [mydumper] -# Local source data directory or the URI of the external storage such as S3. For more information about the URI of the external storage, see https://docs.pingcap.com/tidb/v6.6/backup-and-restore-storages#uri-format. +# Local source data directory or the URI of the external storage such as S3. For more information about the URI of the external storage, see https://docs.pingcap.com/tidb/v7.5/backup-and-restore-storages#uri-format. data-source-dir = "/data/my_database" ``` @@ -28,6 +27,76 @@ When TiDB Lightning is running, it looks for all files that match the pattern of TiDB Lightning processes data in parallel as much as possible. Because files must be read in sequence, the data processing concurrency is at the file level (controlled by `region-concurrency`). Therefore, when the imported file is large, the import performance is poor. It is recommended to limit the size of the imported file to no greater than 256 MiB to achieve the best performance. +## Rename databases and tables + +TiDB Lightning follows filename patterns to import data to the corresponding database and table. If the database or table names change, you can either rename the files and then import them, or use regular expressions to replace the names online. + +### Rename files in batch + +If you are using Red Hat Linux or a distribution based on Red Hat Linux, you can use the `rename` command to batch rename files in the `data-source-dir` directory. + +For example: + +```shell +rename srcdb. tgtdb. *.sql +``` + +After you modify the database name, it is recommended that you delete the `${db_name}-schema-create.sql` file that contains the `CREATE DATABASE` DDL statement from the `data-source-dir` directory. If you want to modify the table name as well, you also need to modify the table name in the `${db_name}.${table_name}-schema.sql` file that contains the `CREATE TABLE` DDL statement. + +### Use regular expressions to replace names online + +To use regular expressions to replace names online, you can use the `pattern` configuration within `[[mydumper.files]]` to match filenames, and replace `schema` and `table` with your desired names. For more information, see [Match customized files](#match-customized-files). + +The following is an example of using regular expressions to replace names online. In this example: + +- The match rule for the data file `pattern` is `^({schema_regrex})\.({table_regrex})\.({file_serial_regrex})\.(csv|parquet|sql)`. +- Specify `schema` as `'$1'`, which means that the value of the first regular expression `schema_regrex` remains unchanged. Or specify `schema` as a string, such as `'tgtdb'`, which means a fixed target database name. +- Specify `table` as `'$2'`, which means that the value of the second regular expression `table_regrex` remains unchanged. Or specify `table` as a string, such as `'t1'`, which means a fixed target table name. +- Specify `type` as `'$3'`, which means the data file type. You can specify `type` as either `"table-schema"` (representing the `schema.sql` file) or `"schema-schema"` (representing the `schema-create.sql` file). + +```toml +[mydumper] +data-source-dir = "/some-subdir/some-database/" +[[mydumper.files]] +pattern = '^(srcdb)\.(.*?)-schema-create\.sql' +schema = 'tgtdb' +type = "schema-schema" +[[mydumper.files]] +pattern = '^(srcdb)\.(.*?)-schema\.sql' +schema = 'tgtdb' +table = '$2' +type = "table-schema" +[[mydumper.files]] +pattern = '^(srcdb)\.(.*?)\.(?:[0-9]+)\.(csv|parquet|sql)' +schema = 'tgtdb' +table = '$2' +type = '$3' +``` + +If you are using `gzip` to back up data files, you need to configure the compression format accordingly. The matching rule of the data file `pattern` is `'^({schema_regrex})\.({table_regrex})\.({file_serial_regrex})\.(csv|parquet|sql)\.(gz)'`. You can specify `compression` as `'$4'` to represent the compressed file format. For example: + +```toml +[mydumper] +data-source-dir = "/some-subdir/some-database/" +[[mydumper.files]] +pattern = '^(srcdb)\.(.*?)-schema-create\.(sql)\.(gz)' +schema = 'tgtdb' +type = "schema-schema" +compression = '$4' +[[mydumper.files]] +pattern = '^(srcdb)\.(.*?)-schema\.(sql)\.(gz)' +schema = 'tgtdb' +table = '$2' +type = "table-schema" +compression = '$4' +[[mydumper.files]] +pattern = '^(srcdb)\.(.*?)\.(?:[0-9]+)\.(sql)\.(gz)' +schema = 'tgtdb' +table = '$2' +type = '$3' +compression = '$4' +``` + ## CSV ### Schema @@ -277,7 +346,7 @@ TiDB Lightning currently only supports Parquet files generated by Amazon Aurora ``` [[mydumper.files]] # The expression needed for parsing Amazon Aurora parquet files -pattern = '(?i)^(?:[^/]*/)*([a-z0-9_]+)\.([a-z0-9_]+)/(?:[^/]*/)*(?:[a-z0-9\-_.]+\.(parquet))$' +pattern = '(?i)^(?:[^/]*/)*([a-z0-9\-_]+).([a-z0-9\-_]+)/(?:[^/]*/)*(?:[a-z0-9\-_.]+\.(parquet))$' schema = '$1' table = '$2' type = '$3' @@ -309,14 +378,14 @@ Take the Aurora snapshot exported to S3 as an example. The complete path of the Usually, `data-source-dir` is set to `S3://some-bucket/some-subdir/some-database/` to import the `some-database` database. -Based on the preceding Parquet file path, you can write a regular expression like `(?i)^(?:[^/]*/)*([a-z0-9_]+)\.([a-z0-9_]+)/(?:[^/]*/)*(?:[a-z0-9\-_.]+\.(parquet))$` to match the files. In the match group, `index=1` is `some-database`, `index=2` is `some-table`, and `index=3` is `parquet`. +Based on the preceding Parquet file path, you can write a regular expression like `(?i)^(?:[^/]*/)*([a-z0-9\-_]+).([a-z0-9\-_]+)/(?:[^/]*/)*(?:[a-z0-9\-_.]+\.(parquet))$` to match the files. In the match group, `index=1` is `some-database`, `index=2` is `some-table`, and `index=3` is `parquet`. You can write the configuration file according to the regular expression and the corresponding index so that TiDB Lightning can recognize the data files that do not follow the default naming convention. For example: ```toml [[mydumper.files]] # The expression needed for parsing the Amazon Aurora parquet file -pattern = '(?i)^(?:[^/]*/)*([a-z0-9_]+)\.([a-z0-9_]+)/(?:[^/]*/)*(?:[a-z0-9\-_.]+\.(parquet))$' +pattern = '(?i)^(?:[^/]*/)*([a-z0-9\-_]+).([a-z0-9\-_]+)/(?:[^/]*/)*(?:[a-z0-9\-_.]+\.(parquet))$' schema = '$1' table = '$2' type = '$3' diff --git a/tidb-lightning/tidb-lightning-distributed-import.md b/tidb-lightning/tidb-lightning-distributed-import.md index 20d6219d0ea09..58d5111dce78a 100644 --- a/tidb-lightning/tidb-lightning-distributed-import.md +++ b/tidb-lightning/tidb-lightning-distributed-import.md @@ -5,7 +5,7 @@ summary: Learn the concept, user scenarios, usages, and limitations of importing # Use TiDB Lightning to Import Data in Parallel -Since v5.3.0, the [physical import mode](/tidb-lightning/tidb-lightning-physical-import-mode.md) of TiDB Lightning supports the parallel import of a single table or multiple tables. By simultaneously running multiple TiDB Lightning instances, you can import data in parallel from different single tables or multiple tables. In this way, TiDB Lightning provides the ability to scale horizontally, which greatly reduces the time required to import large amount of data. +Since v5.3.0, the [physical import mode](/tidb-lightning/tidb-lightning-physical-import-mode.md) of TiDB Lightning supports the parallel import of a single table or multiple tables. By simultaneously running multiple TiDB Lightning instances, you can import data from single or multiple tables in parallel. In this way, TiDB Lightning provides the ability to scale horizontally, which greatly reduces the time required to import large amount of data. In technical implementation, TiDB Lightning records the meta data of each instance and the data of each imported table in the target TiDB, and coordinates the Row ID allocation range of different instances, the record of global Checksum, and the configuration changes and recovery of TiKV and PD. diff --git a/tidb-lightning/tidb-lightning-faq.md b/tidb-lightning/tidb-lightning-faq.md index 5aaf8b97c702c..1bad6afa2ac0b 100644 --- a/tidb-lightning/tidb-lightning-faq.md +++ b/tidb-lightning/tidb-lightning-faq.md @@ -1,7 +1,6 @@ --- title: TiDB Lightning FAQs summary: Learn about the frequently asked questions (FAQs) and answers about TiDB Lightning. -aliases: ['/docs/dev/tidb-lightning/tidb-lightning-faq/','/docs/dev/faq/tidb-lightning/'] --- # TiDB Lightning FAQs diff --git a/tidb-lightning/tidb-lightning-glossary.md b/tidb-lightning/tidb-lightning-glossary.md index 3645dc3997970..0fc8056071e60 100644 --- a/tidb-lightning/tidb-lightning-glossary.md +++ b/tidb-lightning/tidb-lightning-glossary.md @@ -1,7 +1,6 @@ --- title: TiDB Lightning Glossary summary: List of special terms used in TiDB Lightning. -aliases: ['/docs/dev/tidb-lightning/tidb-lightning-glossary/','/docs/dev/reference/tools/tidb-lightning/glossary/'] --- # TiDB Lightning Glossary diff --git a/tidb-lightning/tidb-lightning-logical-import-mode-usage.md b/tidb-lightning/tidb-lightning-logical-import-mode-usage.md index ee2ac84758707..8ac7c933deaef 100644 --- a/tidb-lightning/tidb-lightning-logical-import-mode-usage.md +++ b/tidb-lightning/tidb-lightning-logical-import-mode-usage.md @@ -52,7 +52,7 @@ Conflicting data refers to two or more records with the same data in the PK or U | Strategy | Default behavior of conflicting data | The corresponding SQL statement | | :-- | :-- | :-- | | `"replace"` | Replacing existing data with new data. | `REPLACE INTO ...` | -| `"ignore"` | Keeping existing data and ignoring new data. | `INSERT IGNORE INTO ...` | +| `"ignore"` | Keeping existing data and ignoring new data. | If `conflict.max-record-rows` is greater than 0, `INSERT INTO` is used; if `conflict.max-record-rows` is `0`, `INSERT IGNORE INTO ...` is used | | `"error"` | Pausing the import and reporting an error. | `INSERT INTO ...` | | `""` | TiDB Lightning does not detect or handle conflicting data. If data with primary and unique key conflicts exists, the subsequent step reports an error. | None | diff --git a/tidb-lightning/tidb-lightning-overview.md b/tidb-lightning/tidb-lightning-overview.md index 0e935697d43e5..8b9645e6f4f40 100644 --- a/tidb-lightning/tidb-lightning-overview.md +++ b/tidb-lightning/tidb-lightning-overview.md @@ -1,12 +1,11 @@ --- title: TiDB Lightning Overview summary: Learn about Lightning and the whole architecture. -aliases: ['/docs/dev/tidb-lightning/tidb-lightning-overview/','/docs/dev/reference/tools/tidb-lightning/overview/','/docs/dev/tidb-lightning/tidb-lightning-tidb-backend/','/docs/dev/reference/tools/tidb-lightning/tidb-backend/','/tidb/dev/tidb-lightning-tidb-backend','/docs/dev/loader-overview/','/docs/dev/reference/tools/loader/','/docs/dev/load-misuse-handling/','/docs/dev/reference/tools/error-case-handling/load-misuse-handling/','/tidb/dev/load-misuse-handling','/tidb/dev/loader-overview/','/tidb/dev/tidb-lightning-backends'] --- # TiDB Lightning Overview -[TiDB Lightning](https://github.com/pingcap/tidb/tree/master/br/pkg/lightning) is a tool used for importing data at TB scale to TiDB clusters. It is often used for initial data import to TiDB clusters. +[TiDB Lightning](https://github.com/pingcap/tidb/tree/release-7.5/br/pkg/lightning) is a tool used for importing data at TB scale to TiDB clusters. It is often used for initial data import to TiDB clusters. TiDB Lightning supports the following file formats: @@ -19,7 +18,6 @@ TiDB Lightning can read data from the following sources: - Local - [Amazon S3](/external-storage-uri.md#amazon-s3-uri-format) - [Google Cloud Storage](/external-storage-uri.md#gcs-uri-format) -- [Azure Blob Storage](/external-storage-uri.md#azure-blob-storage-uri-format) ## TiDB Lightning architecture diff --git a/tidb-lightning/tidb-lightning-physical-import-mode-usage.md b/tidb-lightning/tidb-lightning-physical-import-mode-usage.md index 5918ebf39f9cf..89a8b0424ac54 100644 --- a/tidb-lightning/tidb-lightning-physical-import-mode-usage.md +++ b/tidb-lightning/tidb-lightning-physical-import-mode-usage.md @@ -206,6 +206,25 @@ store-write-bwlimit = "128MiB" distsql-scan-concurrency = 3 ``` +You can measure the impact of data import on TPCC results by simulating the online application using TPCC and importing data into a TiDB cluster using TiDB Lightning. The test result is as follows: + +| Concurrency | TPM | P99 | P90 | AVG | +| ----- | --- | --- | --- | --- | +| 1 | 20%~30% | 60%~80% | 30%~50% | 30%~40% | +| 8 | 15%~25% | 70%~80% | 35%~45% | 20%~35% | +| 16 | 20%~25% | 55%~85% | 35%~40% | 20%~30% | +| 64 | No significant impact | +| 256 | No significant impact | + +The percentage in the preceding table indicates the impact of data import on TPCC results. + +* For the TPM column, the number indicates the percentage of TPM decrease. +* For the P99, P90, and AVG columns, the number indicates the percentage of latency increase. + +The test results show that the smaller the concurrency, the larger the impact of data import on TPCC results. When the concurrency is 64 or more, the impact of data import on TPCC results is negligible. + +Therefore, if your TiDB cluster has a latency-sensitive application and a low concurrency, it is strongly recommended **not** to use TiDB Lightning to import data into the cluster. This will cause a significant impact on the online application. + ## Performance tuning **The most direct and effective ways to improve import performance of the physical import mode are as follows:** diff --git a/tidb-lightning/tidb-lightning-physical-import-mode.md b/tidb-lightning/tidb-lightning-physical-import-mode.md index f7ff899fe05c3..78a0e26d450c3 100644 --- a/tidb-lightning/tidb-lightning-physical-import-mode.md +++ b/tidb-lightning/tidb-lightning-physical-import-mode.md @@ -5,7 +5,7 @@ summary: Learn about the physical import mode in TiDB Lightning. # Physical Import Mode -Physical import mode is an efficient and fast import mode that inserts data directly into TiKV nodes as key-value pairs without going through the SQL interface. When using the physical import mode, a single instance of Lightning can import up to 10 TiB of data. The supported amount of imported data theoretically increases as the number of Lightning instances increases. It is verified by users that [parallel importing](/tidb-lightning/tidb-lightning-distributed-import.md) based on Lightning can effectively handle up to 20 TiB of data. +Physical import mode is an efficient and fast import mode that inserts data directly into TiKV nodes as key-value pairs without going through the SQL interface. When using the physical import mode, a single instance of Lightning can import up to 10 TiB of data. The supported amount of imported data theoretically increases as the number of Lightning instances increases. It is verified by users that [parallel importing](/tidb-lightning/tidb-lightning-distributed-import.md) based on Lightning can effectively handle up to 50 TiB of data. Before you use the physical import mode, make sure to read [Requirements and restrictions](#requirements-and-restrictions). @@ -21,7 +21,7 @@ The backend for the physical import mode is `local`. You can modify it in `tidb- 1. Before importing data, TiDB Lightning automatically switches the TiKV nodes to "import mode", which improves write performance and stops auto-compaction. TiDB Lightning determines whether to pause global scheduling according to the TiDB Lightning version. - - Starting from v7.1.0, you can you can control the scope of pausing scheduling by using the TiDB Lightning parameter [`pause-pd-scheduler-scope`](/tidb-lightning/tidb-lightning-configuration.md). + - Starting from v7.1.0, you can control the scope of pausing scheduling by using the TiDB Lightning parameter [`pause-pd-scheduler-scope`](/tidb-lightning/tidb-lightning-configuration.md). - For TiDB Lightning versions between v6.2.0 and v7.0.0, the behavior of pausing global scheduling depends on the TiDB cluster version. When the TiDB cluster >= v6.1.0, TiDB Lightning pauses scheduling for the Region that stores the target table data. After the import is completed, TiDB Lightning recovers scheduling. For other versions, TiDB Lightning pauses global scheduling. - When TiDB Lightning < v6.2.0, TiDB Lightning pauses global scheduling. @@ -74,10 +74,10 @@ It is recommended that you allocate CPU more than 32 cores and memory greater th - Do not use the physical import mode to directly import data to TiDB clusters in production. It has severe performance implications. If you need to do so, refer to [Pause scheduling on the table level](/tidb-lightning/tidb-lightning-physical-import-mode-usage.md#scope-of-pausing-scheduling-during-import). - If your TiDB cluster has a latency-sensitive application and a low concurrency, it is strongly recommended that you **do not** use the physical import mode to import data into the cluster. This mode might have significant impact on the online application. -- Do not use multiple TiDB Lightning instances to import data to the same TiDB cluster by default. Use [Parallel Import](/tidb-lightning/tidb-lightning-distributed-import.md) instead. +- If you want to run multiple TiDB Lightning instances simultaneously to import data into the same TiDB cluster, you can enable [Parallel Import](/tidb-lightning/tidb-lightning-distributed-import.md) to coordinate the import. If each instance imports data into **different tables**, the Parallel Import option is not required. However, if multiple instances import data into **the same table**, you need to enable Parallel Import to prevent conflicts and ensure data integrity. - When you use multiple TiDB Lightning to import data to the same target cluster, do not mix the import modes. That is, do not use the physical import mode and the logical import mode at the same time. - During the process of importing data, do not perform DDL and DML operations in the target table. Otherwise the import will fail or the data will be inconsistent. At the same time, it is not recommended to perform read operations, because the data you read might be inconsistent. You can perform read and write operations after the import operation is completed. -- A single Lightning process can import a single table of 10 TB at most. Parallel import can use 10 Lightning instances at most. +- A single Lightning process can import a single table of 10 TiB at most. Parallel import can use 10 Lightning instances at most. ### Tips for using with other components @@ -92,3 +92,9 @@ It is recommended that you allocate CPU more than 32 cores and memory greater th - When you use TiDB Lightning with TiCDC, note the following: - TiCDC cannot capture the data inserted in the physical import mode. + +- When you use TiDB Lightning with BR, note the following: + + - When BR backs up snapshots of tables that are being imported by TiDB Lightning, it might result in inconsistent backup data for those tables. + - When BR backs up data using AWS EBS volume snapshots, TiDB Lightning might fail to import data. + - Data imported in TiDB Lightning physical import mode does not support [log backup](/br/br-pitr-guide.md#start-log-backup) and thereby cannot be restored by Point-in-Time Recovery (PITR). \ No newline at end of file diff --git a/tidb-lightning/tidb-lightning-prechecks.md b/tidb-lightning/tidb-lightning-prechecks.md index c5ae306246c71..2063523d92e77 100644 --- a/tidb-lightning/tidb-lightning-prechecks.md +++ b/tidb-lightning/tidb-lightning-prechecks.md @@ -14,8 +14,9 @@ The following table describes each check item and detailed explanation. | Cluster version and status| >= 5.3.0 | Check whether the cluster can be connected in the configuration, and whether the TiKV/PD/TiFlash version supports the physical import mode. | | Permissions | >= 5.3.0 | When the data source is cloud storage (Amazon S3), check whether TiDB Lightning has the necessary permissions and make sure that the import will not fail due to lack of permissions. | | Disk space | >= 5.3.0 | Check whether there is enough space on the local disk and on the TiKV cluster for importing data. TiDB Lightning samples the data sources and estimates the percentage of the index size from the sample result. Because indexes are included in the estimation, there might be cases where the size of the source data is less than the available space on the local disk, but still, the check fails. In the physical import mode, TiDB Lightning also checks whether the local storage is sufficient because external sorting needs to be done locally. For more details about the TiKV cluster space and local storage space (controlled by `sort-kv-dir`), see [Downstream storage space requirements](/tidb-lightning/tidb-lightning-requirements.md#storage-space-of-the-target-database) and [Resource requirements](/tidb-lightning/tidb-lightning-physical-import-mode.md#environment-requirements). | -| Region distribution status | >= 5.3.0 | Check whether the Regions in the TiKV cluster are distributed evenly and whether there are too many empty Regions. If the number of empty Regions exceeds max(1000, number of tables * 3), i.e. greater than the bigger one of "1000" or "3 times the number of tables ", then the import cannot be executed. | +| Region distribution status | >= 5.3.0 | Check whether the Regions in the TiKV cluster are distributed evenly and whether there are too many empty Regions. If the number of empty Regions exceeds max(1000, number of tables * 3), i.e. greater than the bigger one of "1000" or "3 times the number of tables", then the import cannot be executed. | | Exceedingly Large CSV files in the data file | >= 5.3.0 | When there are CSV files larger than 10 GiB in the backup file and auto-slicing is not enabled (StrictFormat=false), it will impact the import performance. The purpose of this check is to remind you to ensure the data is in the right format and to enable auto-slicing. | | Recovery from breakpoints | >= 5.3.0 | This check ensures that no changes are made to the source file or schema in the database during the breakpoint recovery process that would result in importing the wrong data. | | Import into an existing table | >= 5.3.0 | When importing into an already created table, it checks, as much as possible, whether the source file matches the existing table. Check if the number of columns matches. If the source file has column names, check if the column names match. When there are default columns in the source file, it checks if the default columns have Default Value, and if they have, the check passes. | | Whether the target table is empty | >= 5.3.1 | TiDB Lightning automatically exits with an error if the target table is not empty. If parallel import mode is enabled (`parallel-import = true`), this check item will be skipped. | +| Whether PITR is enabled or any changefeed task is running in the cluster | >= 6.5.0 | The TiDB Lightning physical import mode is incompatible with PITR and changefeed. If PITR is enabled or a changefeed task is running, TiDB Lightning automatically exits with an error. If you are certain that the tables to be imported do not require PITR or changefeed for replication, you can skip this check to proceed with the TiDB Lightning physical import mode. | diff --git a/tidb-lightning/tidb-lightning-requirements.md b/tidb-lightning/tidb-lightning-requirements.md index c7897a9794aba..a866870437871 100644 --- a/tidb-lightning/tidb-lightning-requirements.md +++ b/tidb-lightning/tidb-lightning-requirements.md @@ -84,18 +84,33 @@ The target TiKV cluster must have enough disk space to store the imported data. - Indexes might take extra space. - RocksDB has a space amplification effect. -It is difficult to calculate the exact data volume exported by Dumpling from MySQL. However, you can estimate the data volume by using the following SQL statement to summarize the data-length field in the information_schema.tables table: - -Calculate the size of all schemas, in MiB. Replace ${schema_name} with your schema name. +It is difficult to calculate the exact data volume exported by Dumpling from MySQL. However, you can estimate the data volume by using the following SQL statement to summarize the `DATA_LENGTH` field in the information_schema.tables table: ```sql -SELECT table_schema, SUM(data_length)/1024/1024 AS data_length, SUM(index_length)/1024/1024 AS index_length, SUM(data_length+index_length)/1024/1024 AS sum FROM information_schema.tables WHERE table_schema = "${schema_name}" GROUP BY table_schema; -``` - -Calculate the size of the largest table, in MiB. Replace ${schema_name} with your schema name. +-- Calculate the size of all schemas +SELECT + TABLE_SCHEMA, + FORMAT_BYTES(SUM(DATA_LENGTH)) AS 'Data Size', + FORMAT_BYTES(SUM(INDEX_LENGTH)) 'Index Size' +FROM + information_schema.tables +GROUP BY + TABLE_SCHEMA; -{{< copyable "sql" >}} - -```sql -SELECT table_name, table_schema, SUM(data_length)/1024/1024 AS data_length, SUM(index_length)/1024/1024 AS index_length,sum(data_length+index_length)/1024/1024 AS sum FROM information_schema.tables WHERE table_schema = "${schema_name}" GROUP BY table_name,table_schema ORDER BY sum DESC LIMIT 5; +-- Calculate the 5 largest tables +SELECT + TABLE_NAME, + TABLE_SCHEMA, + FORMAT_BYTES(SUM(data_length)) AS 'Data Size', + FORMAT_BYTES(SUM(index_length)) AS 'Index Size', + FORMAT_BYTES(SUM(data_length+index_length)) AS 'Total Size' +FROM + information_schema.tables +GROUP BY + TABLE_NAME, + TABLE_SCHEMA +ORDER BY + SUM(DATA_LENGTH+INDEX_LENGTH) DESC +LIMIT + 5; ``` diff --git a/tidb-lightning/tidb-lightning-web-interface.md b/tidb-lightning/tidb-lightning-web-interface.md index ff853bb960ef9..9836257ffbfd3 100644 --- a/tidb-lightning/tidb-lightning-web-interface.md +++ b/tidb-lightning/tidb-lightning-web-interface.md @@ -1,7 +1,6 @@ --- title: TiDB Lightning Web Interface summary: Control TiDB Lightning through the web interface. -aliases: ['/docs/dev/tidb-lightning/tidb-lightning-web-interface/','/docs/dev/reference/tools/tidb-lightning/web/'] --- # TiDB Lightning Web Interface diff --git a/tidb-lightning/troubleshoot-tidb-lightning.md b/tidb-lightning/troubleshoot-tidb-lightning.md index cc7adf0855c6e..7c9076a86be86 100644 --- a/tidb-lightning/troubleshoot-tidb-lightning.md +++ b/tidb-lightning/troubleshoot-tidb-lightning.md @@ -1,7 +1,6 @@ --- title: Troubleshoot TiDB Lightning summary: Learn the common problems you might encounter when you use TiDB Lightning and their solutions. -aliases: ['/docs/dev/troubleshoot-tidb-lightning/','/docs/dev/how-to/troubleshoot/tidb-lightning/','/docs/dev/tidb-lightning/tidb-lightning-misuse-handling/','/docs/dev/reference/tools/error-case-handling/lightning-misuse-handling/','/tidb/dev/tidb-lightning-misuse-handling','/tidb/dev/troubleshoot-tidb-lightning'] --- # Troubleshoot TiDB Lightning @@ -47,7 +46,7 @@ It is potentially caused by starting `tidb-lightning` incorrectly, which causes [2018/08/10 07:29:08.310 +08:00] [INFO] [main.go:41] ["got signal to exit"] [signal=hangup] ``` -It is not recommended to directly use `nohup` in the command line to start `tidb-lightning`. You can [start `tidb-lightning`](/tidb-lightning/deploy-tidb-lightning.md#step-3-start-tidb-lightning) by executing a script. +It is not recommended to directly use `nohup` in the command line to start `tidb-lightning`. You can [start `tidb-lightning`](/get-started-with-tidb-lightning.md#step-4-start-tidb-lightning) by executing a script. In addition, if the last log of TiDB Lightning shows that the error is "Context canceled", you need to search for the first "ERROR" level log. This "ERROR" level log is usually followed by "got signal to exit", which indicates that TiDB Lightning received an interrupt signal and then exited. diff --git a/tidb-limitations.md b/tidb-limitations.md index 519ad878c201f..b724f8fb1c2d7 100644 --- a/tidb-limitations.md +++ b/tidb-limitations.md @@ -1,13 +1,16 @@ --- title: TiDB Limitations summary: Learn the usage limitations of TiDB. -aliases: ['/docs/dev/tidb-limitations/'] --- # TiDB Limitations This document describes the common usage limitations of TiDB, including the maximum identifier length and the maximum number of supported databases, tables, indexes, partitioned tables, and sequences. +> **Note:** +> +> TiDB offers high compatibility with the MySQL protocol and syntax, including many MySQL limitations. For example, a single index can include a maximum of 16 columns. For more information, see [MySQL Compatibility](/mysql-compatibility.md) and the official MySQL documentation. + ## Limitations on identifier length | Identifier type | Maximum length (number of characters allowed) | @@ -59,7 +62,7 @@ This document describes the common usage limitations of TiDB, including the maxi -You can adjust the size limit via the [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50) configuration item. +You can adjust the size limit via the [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v4010-and-v500) configuration item. @@ -67,12 +70,12 @@ You can adjust the size limit via the [`txn-entry-size-limit`](/tidb-configurati | Type | Upper limit | |:----------|:----------| -| CHAR | 256 characters | -| BINARY | 256 characters | -| VARBINARY | 65535 characters | +| CHAR | 255 characters | +| BINARY | 255 bytes | +| VARBINARY | 65535 bytes | | VARCHAR | 16383 characters | -| TEXT | Defaults to 6 MiB and can be adjusted to 120 MiB | -| BLOB | Defaults to 6 MiB and can be adjusted to 120 MiB | +| TEXT | 65535 bytes | +| BLOB | 65535 bytes | ## Limitations on SQL statements diff --git a/tidb-monitoring-api.md b/tidb-monitoring-api.md index 04f16150775b3..337b4cd8eff25 100644 --- a/tidb-monitoring-api.md +++ b/tidb-monitoring-api.md @@ -1,7 +1,6 @@ --- title: TiDB Monitoring API summary: Learn the API of TiDB monitoring services. -aliases: ['/docs/dev/tidb-monitoring-api/'] --- # TiDB Monitoring API @@ -28,7 +27,7 @@ The following example uses `http://${host}:${port}/status` to get the current st curl http://127.0.0.1:10080/status { connections: 0, # The current number of clients connected to the TiDB server. - version: "8.0.11-TiDB-v7.4.0", # The TiDB version number. + version: "8.0.11-TiDB-v{{{ .tidb-version }}}", # The TiDB version number. git_hash: "778c3f4a5a716880bcd1d71b257c8165685f0d70" # The Git Hash of the current TiDB code. } ``` @@ -79,7 +78,7 @@ curl http://127.0.0.1:10080/schema_storage/test - PD API address: `http://${host}:${port}/pd/api/v1/${api_name}` - Default port: `2379` -- Details about API names: see [PD API doc](https://download.pingcap.com/pd-api-v1.html) +- Details about API names: see [PD API doc](https://docs-download.pingcap.com/api/pd-api/pd-api-v1.html) The PD interface provides the status of all the TiKV servers and the information about load balancing. See the following example for the information about a single-node TiKV cluster: diff --git a/tidb-monitoring-framework.md b/tidb-monitoring-framework.md index 04d0bbcf333b9..e10d7faa4e17b 100644 --- a/tidb-monitoring-framework.md +++ b/tidb-monitoring-framework.md @@ -1,12 +1,11 @@ --- title: TiDB Monitoring Framework Overview -summary: Use Prometheus and Grafana to build the TiDB monitoring framework. -aliases: ['/docs/dev/tidb-monitoring-framework/','/docs/dev/how-to/monitor/overview/'] +summary: Use Prometheus, Grafana, and TiDB Dashboard to build the TiDB monitoring framework. --- # TiDB Monitoring Framework Overview -The TiDB monitoring framework adopts two open source projects: Prometheus and Grafana. TiDB uses [Prometheus](https://prometheus.io) to store the monitoring and performance metrics and [Grafana](https://grafana.com/grafana) to visualize these metrics. +The TiDB monitoring framework adopts two open source projects: Prometheus and Grafana. TiDB uses [Prometheus](https://prometheus.io) to store the monitoring and performance metrics and [Grafana](https://grafana.com/grafana) to visualize these metrics. TiDB also provides a built-in [TiDB Dashboard](/dashboard/dashboard-intro.md) for monitoring and diagnosing TiDB clusters. ## About Prometheus in TiDB @@ -51,3 +50,7 @@ Grafana is an open source project for analyzing and visualizing metrics. TiDB us Each group has multiple panel labels of monitoring metrics, and each panel contains detailed information of multiple monitoring metrics. For example, the **Overview** monitoring group has five panel labels, and each labels corresponds to a monitoring panel. See the following UI: ![Grafana Overview](/media/grafana-monitor-overview.png) + +## TiDB Dashboard + +TiDB Dashboard is a web UI for monitoring, diagnosing, and managing the TiDB cluster, which is introduced in v4.0. It is built into the PD component and does not require an independent deployment. For more information, see [TiDB Dashboard introduction](/dashboard/dashboard-intro.md). diff --git a/tidb-operator-overview.md b/tidb-operator-overview.md index e03a0119e2c6b..901e5e2c0b0e2 100644 --- a/tidb-operator-overview.md +++ b/tidb-operator-overview.md @@ -1,7 +1,6 @@ --- title: TiDB Operator summary: Learn about TiDB Operator, the automatic operation system for TiDB clusters on Kubernetes. -aliases: ['/docs/tidb-in-kubernetes/dev/'] --- # TiDB Operator diff --git a/tidb-resource-control.md b/tidb-resource-control.md index d6f6358be9d2e..c88e814613e2b 100644 --- a/tidb-resource-control.md +++ b/tidb-resource-control.md @@ -7,7 +7,7 @@ summary: Learn how to use the resource control feature to control and schedule a > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) and [{{{ .essential }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#essential) clusters. As a cluster administrator, you can use the resource control feature to create resource groups, set quotas for resource groups, and bind users to those groups. @@ -46,6 +46,10 @@ With this feature, you can: In addition, the rational use of the resource control feature can reduce the number of clusters, ease the difficulty of operation and maintenance, and save management costs. +> **Note:** +> +> - To assess the effectiveness of resource management, it is recommended to deploy the cluster on independent computing and storage nodes. Scheduling and other cluster resource-sensitive features are hardly working properly on the deployment created by `tiup playground`, where the resources are shared across instances. + ## Limitations Resource control incurs additional scheduling overhead. Therefore, there might be a slight performance degradation (less than 5%) when this feature is enabled. @@ -74,7 +78,7 @@ Request Unit (RU) is a unified abstraction unit in TiDB for system resources, wh Write - 1 storage write batch consumes 1 RU for each replica + 1 storage write batch consumes 1 RU 1 storage write request consumes 1 RU @@ -83,23 +87,17 @@ Request Unit (RU) is a unified abstraction unit in TiDB for system resources, wh 1 KiB write request payload consumes 1 RU - SQL CPU + CPU 3 ms consumes 1 RU -Currently, TiFlash resource control only considers SQL CPU, which is the CPU time consumed by the execution of pipeline tasks for queries, and read request payload. - > **Note:** > > - Each write operation is eventually replicated to all replicas (by default, TiKV has 3 replicas). Each replication operation is considered a different write operation. -> - In addition to queries executed by users, RU can be consumed by background tasks, such as automatic statistics collection. -> - The preceding table lists only the resources involved in RU calculation for TiDB Self-Hosted clusters, excluding the network and storage consumption. For TiDB Serverless RUs, see [TiDB Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details/). - -## Estimate RU consumption of SQL statements - -You can use the [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md#ru-request-unit-consumption) statement to get the amount of RUs consumed during SQL execution. Note that the amount of RUs is affected by the cache (for example, [coprocessor cache](/coprocessor-cache.md)). When the same SQL is executed multiple times, the amount of RUs consumed by each execution might be different. The RU value does not represent the exact value for each execution, but can be used as a reference for estimation. +> - The preceding table lists only the resources involved in RU calculation for TiDB Self-Managed clusters, excluding the network and storage consumption. For {{{ .starter }}} RUs, see [{{{ .starter }}} Pricing Details](https://www.pingcap.com/tidb-cloud-starter-pricing-details/). +> - Currently, TiFlash resource control only considers SQL CPU, which is the CPU time consumed by the execution of pipeline tasks for queries, and read request payload. ## Parameters for resource control @@ -116,8 +114,8 @@ The resource control feature introduces the following system variables or parame -* TiKV: For TiDB Self-Hosted, you can use the `resource-control.enabled` parameter to control whether to use request scheduling based on resource group quotas. For TiDB Cloud, the value of the `resource-control.enabled` parameter is `true` by default and does not support dynamic modification. -* TiFlash: For TiDB Self-Hosted, you can use the `tidb_enable_resource_control` system variable and the `enable_resource_control` configuration item (introduced in v7.4.0) to control whether to enable TiFlash resource control. +* TiKV: For TiDB Self-Managed, you can use the `resource-control.enabled` parameter to control whether to use request scheduling based on resource group quotas. For TiDB Cloud, the value of the `resource-control.enabled` parameter is `true` by default and does not support dynamic modification. +* TiFlash: For TiDB Self-Managed, you can use the `tidb_enable_resource_control` system variable and the `enable_resource_control` configuration item (introduced in v7.4.0) to control whether to enable TiFlash resource control. @@ -140,7 +138,7 @@ Starting from v7.4.0, the TiFlash configuration item `enable_resource_control` i -For more information about the resource control mechanism and parameters, see [RFC: Global Resource Control in TiDB](https://github.com/pingcap/tidb/blob/master/docs/design/2022-11-25-global-resource-control.md) and [TiFlash Resource Control](https://github.com/pingcap/tiflash/blob/master/docs/design/2023-09-21-tiflash-resource-control.md). +For more information about the resource control mechanism and parameters, see [RFC: Global Resource Control in TiDB](https://github.com/pingcap/tidb/blob/release-7.5/docs/design/2022-11-25-global-resource-control.md) and [TiFlash Resource Control](https://github.com/pingcap/tiflash/blob/release-7.5/docs/design/2023-09-21-tiflash-resource-control.md). ## How to use resource control @@ -161,9 +159,9 @@ You can view the [Resource Manager page](/dashboard/dashboard-resource-manager.m -For TiDB Self-Hosted, you can use the [`CALIBRATE RESOURCE`](https://docs.pingcap.com/zh/tidb/stable/sql-statement-calibrate-resource) statement to estimate the cluster capacity. +For TiDB Self-Managed, you can use the [`CALIBRATE RESOURCE`](https://docs.pingcap.com/tidb/stable/sql-statement-calibrate-resource) statement to estimate the cluster capacity. -For TiDB Cloud, the [`CALIBRATE RESOURCE`](https://docs.pingcap.com/zh/tidb/stable/sql-statement-calibrate-resource) statement is inapplicable. +For TiDB Cloud, the [`CALIBRATE RESOURCE`](https://docs.pingcap.com/tidb/stable/sql-statement-calibrate-resource) statement is inapplicable. @@ -264,7 +262,7 @@ SELECT /*+ RESOURCE_GROUP(rg1) */ * FROM t limit 10; > > This feature is experimental. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. -A runaway query is a query that consumes more time or resources than expected. The term **runaway queries** is used in the following to describe the feature of managing the runaway query. +A runaway query is a query (`SELECT` statement only) that consumes more time or resources than expected. The term **runaway queries** is used in the following to describe the feature of managing the runaway query. - Starting from v7.2.0, the resource control feature introduces the management of runaway queries. You can set criteria for a resource group to identify runaway queries and automatically take actions to prevent them from exhausting resources and affecting other queries. You can manage runaway queries for a resource group by including the `QUERY_LIMIT` field in [`CREATE RESOURCE GROUP`](/sql-statements/sql-statement-create-resource-group.md) or [`ALTER RESOURCE GROUP`](/sql-statements/sql-statement-alter-resource-group.md). - Starting from v7.3.0, the resource control feature introduces manual management of runaway watches, enabling quick identification of runaway queries for a given SQL statement or Digest. You can execute the statement [`QUERY WATCH`](/sql-statements/sql-statement-query-watch.md) to manually manage the runaway queries watch list in the resource group. @@ -286,7 +284,7 @@ To avoid too many concurrent runaway queries that exhaust system resources, the There are three methods for `WATCH` to match for quick identification: - `EXACT` indicates that only SQL statements with exactly the same SQL text are quickly identified. -- `SIMILAR` indicates all SQL statements with the same pattern are matched by Plan Digest, and the literal values are ignored. +- `SIMILAR` indicates all SQL statements with the same pattern are matched by SQL Digest, and the literal values are ignored. - `PLAN` indicates all SQL statements with the same pattern are matched by Plan Digest. The `DURATION` option in `WATCH` indicates the duration of the identification item, which is infinite by default. @@ -382,7 +380,7 @@ You can get more information about runaway queries from the following system tab + The `mysql.tidb_runaway_queries` table contains the history records of all runaway queries identified in the past 7 days. Take one of the rows as an example: ```sql - MySQL [(none)]> SELECT * FROM mysql.tidb_runaway_queries LIMIT 1\G; + MySQL [(none)]> SELECT * FROM mysql.tidb_runaway_queries LIMIT 1\G *************************** 1. row *************************** resource_group_name: rg1 time: 2023-06-16 17:40:22 @@ -405,6 +403,8 @@ You can get more information about runaway queries from the following system tab > **Warning:** > > This feature is experimental. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://docs.pingcap.com/tidb/stable/support) on GitHub. +> +> The background task management in resource control is based on TiKV's dynamic adjustment of resource quotas for CPU/IO utilization. Therefore, it relies on the available resource quota of each instance. If multiple components or instances are deployed on a single server, it is mandatory to set the appropriate resource quota for each instance through `cgroup`. It is difficult to achieve the expected effect in deployment with shared resources like TiUP Playground. Background tasks, such as data backup and automatic statistics collection, are low-priority but consume many resources. These tasks are usually triggered periodically or irregularly. During execution, they consume a lot of resources, thus affecting the performance of online high-priority tasks. @@ -422,6 +422,7 @@ TiDB supports the following types of background tasks: - `br`: perform backup and restore tasks using [BR](/br/backup-and-restore-overview.md). PITR is not supported. - `ddl`: control the resource usage during the batch data write back phase of Reorg DDLs. - `stats`: the [collect statistics](/statistics.md#collect-statistics) tasks that are manually executed or automatically triggered by TiDB. +- `background`: a reserved task type. You can use the [`tidb_request_source_type`](/system-variables.md#tidb_request_source_type-new-in-v740) system variable to specify the task type of the current session as `background`. @@ -431,35 +432,60 @@ TiDB supports the following types of background tasks: - `br`: perform backup and restore tasks using [BR](https://docs.pingcap.com/tidb/stable/backup-and-restore-overview). PITR is not supported. - `ddl`: control the resource usage during the batch data write back phase of Reorg DDLs. - `stats`: the [collect statistics](/statistics.md#collect-statistics) tasks that are manually executed or automatically triggered by TiDB. +- `background`: a reserved task type. You can use the [`tidb_request_source_type`](/system-variables.md#tidb_request_source_type-new-in-v740) system variable to specify the task type of the current session as `background`. -By default, the task types that are marked as background tasks are empty, and the management of background tasks is disabled. This default behavior is the same as that of versions prior to TiDB v7.4.0. To manage background tasks, you need to manually modify the background task types of the `default` resource group. +By default, the task types that are marked as background tasks are `""`, and the management of background tasks is disabled. To enable background task management, you need to manually modify the background task type of the `default` resource group. After a background task is identified and matched, Resource Control is automatically performed. This means that when system resources are insufficient, the background tasks are automatically reduced to the lowest priority to ensure the execution of foreground tasks. + +> **Note:** +> +> Currently, background tasks for all resource groups are bound to the `default` resource group. You can manage background task types globally through `default`. Binding background tasks to other resource groups is currently not supported. #### Examples -1. Create the `rg1` resource group and set `br` and `stats` as background tasks. +1. Modify the `default` resource group and mark `br` and `ddl` as background tasks. ```sql - CREATE RESOURCE GROUP IF NOT EXISTS rg1 RU_PER_SEC = 500 BACKGROUND=(TASK_TYPES='br,stats'); + ALTER RESOURCE GROUP `default` BACKGROUND=(TASK_TYPES='br,ddl'); ``` -2. Change the `rg1` resource group to set `br` and `ddl` as background tasks. +2. Change the `default` resource group to revert the background task type to its default value. ```sql - ALTER RESOURCE GROUP rg1 BACKGROUND=(TASK_TYPES='br,ddl'); + ALTER RESOURCE GROUP `default` BACKGROUND=NULL; ``` -3. Restore the background tasks of `rg1` resource group to the default value. In this case, the background task types follow the configuration of the `default` resource group. +3. Change the `default` resource group to set the background task type to empty. In this case, all tasks of this resource group are not treated as background tasks. ```sql - ALTER RESOURCE GROUP rg1 BACKGROUND=NULL; + ALTER RESOURCE GROUP `default` BACKGROUND=(TASK_TYPES=""); ``` -4. Change the `rg1` resource group to set the background task types to empty. In this case, all tasks of this resource group are not treated as background tasks. +4. View the background task type of the `default` resource group. ```sql - ALTER RESOURCE GROUP rg1 BACKGROUND=(TASK_TYPES=""); + SELECT * FROM information_schema.resource_groups WHERE NAME="default"; + ``` + + The output is as follows: + + ``` + +---------+------------+----------+-----------+-------------+---------------------+ + | NAME | RU_PER_SEC | PRIORITY | BURSTABLE | QUERY_LIMIT | BACKGROUND | + +---------+------------+----------+-----------+-------------+---------------------+ + | default | UNLIMITED | MEDIUM | YES | NULL | TASK_TYPES='br,ddl' | + +---------+------------+----------+-----------+-------------+---------------------+ + ``` + +5. To explicitly mark tasks in the current session as the background type, you can use `tidb_request_source_type` to explicitly specify the task type. The following is an example: + + ``` sql + SET @@tidb_request_source_type="background"; + /* Add background task type */ + ALTER RESOURCE GROUP `default` BACKGROUND=(TASK_TYPES="background"); + /* Execute LOAD DATA in the current session */ + LOAD DATA INFILE "s3://resource-control/Lightning/test.customer.aaaa.csv" ``` ## Disable resource control @@ -486,12 +512,81 @@ By default, the task types that are marked as background tasks are empty, and th SET GLOBAL tidb_enable_resource_control = 'OFF'; ``` -2. For TiDB Self-Hosted, you can use the `resource-control.enabled` parameter to control whether to use request scheduling based on resource group quotas. For TiDB Cloud, the value of the `resource-control.enabled` parameter is `true` by default and does not support dynamic modification. If you need to disable it for TiDB Dedicated clusters, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). +2. For TiDB Self-Managed, you can use the `resource-control.enabled` parameter to control whether to use request scheduling based on resource group quotas. For TiDB Cloud, the value of the `resource-control.enabled` parameter is `true` by default and does not support dynamic modification. If you need to disable it for TiDB Cloud Dedicated clusters, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). + +3. For TiDB Self-Managed, you can use the `enable_resource_control` configuration item to control whether to enable TiFlash resource control. For TiDB Cloud, the value of the `enable_resource_control` parameter is `true` by default and does not support dynamic modification. If you need to disable it for TiDB Cloud Dedicated clusters, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). + + + +## View RU consumption + +You can view information about RU consumption. + +### View the RU consumption by SQL + +You can view the RU consumption of SQL statements in the following ways: + +- The system variable `tidb_last_query_info` +- `EXPLAIN ANALYZE` +- Slow queries and corresponding system table +- `statements_summary` + +#### View the RUs consumed by the last SQL execution by querying the system variable `tidb_last_query_info` + +TiDB provides the system variable [`tidb_last_query_info`](/system-variables.md#tidb_last_query_info-new-in-v4014). This system variable records the information of the last DML statement executed, including the RUs consumed by the SQL execution. + +Example: + +1. Run the `UPDATE` statement: + + ```sql + UPDATE sbtest.sbtest1 SET k = k + 1 WHERE id = 1; + ``` + + ``` + Query OK, 1 row affected (0.01 sec) + Rows matched: 1 Changed: 1 Warnings: 0 + ``` + +2. Query the system variable `tidb_last_query_info` to view the information of the last executed statement: + + ```sql + SELECT @@tidb_last_query_info; + ``` + + ``` + +------------------------------------------------------------------------------------------------------------------------+ + | @@tidb_last_query_info | + +------------------------------------------------------------------------------------------------------------------------+ + | {"txn_scope":"global","start_ts":446809472210829315,"for_update_ts":446809472210829315,"ru_consumption":4.34885578125} | + +------------------------------------------------------------------------------------------------------------------------+ + 1 row in set (0.01 sec) + ``` + + In the result, `ru_consumption` is the RUs consumed by the execution of this SQL statement. + +#### View RUs consumed during SQL execution by `EXPLAIN ANALYZE` + +You can use the [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md#ru-request-unit-consumption) statement to get the amount of RUs consumed during SQL execution. Note that the amount of RUs is affected by the cache (for example, [coprocessor cache](/coprocessor-cache.md)). When the same SQL is executed multiple times, the amount of RUs consumed by each execution might be different. The RU value does not represent the exact value for each execution, but can be used as a reference for estimation. + +#### Slow queries and the corresponding system table + + + +When you enable resource control, the [slow query log](/identify-slow-queries.md) of TiDB and the corresponding system table [`INFORMATION_SCHEMA.SLOW_QUERY`](/information-schema/information-schema-slow-query.md) contain the resource group, RU consumption of the corresponding SQL, and the time spent waiting for available RUs. + + -3. For TiDB Self-Hosted, you can use the `enable_resource_control` configuration item to control whether to enable TiFlash resource control. For TiDB Cloud, the value of the `enable_resource_control` parameter is `true` by default and does not support dynamic modification. If you need to disable it for TiDB Dedicated clusters, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). + + +When you enable resource control, the system table [`INFORMATION_SCHEMA.SLOW_QUERY`](/information-schema/information-schema-slow-query.md) contains the resource group, RU consumption of the corresponding SQL, and the time spent waiting for available RUs. +#### View RU statistics by `statements_summary` + +The system table [`INFORMATION_SCHEMA.statements_summary`](/statement-summary-tables.md#statements_summary) in TiDB stores the normalized and aggregated statistics of SQL statements. You can use the system table to view and analyze the execution performance of SQL statements. It also contains statistics about resource control, including the resource group name, RU consumption, and the time spent waiting for available RUs. For more details, see [`statements_summary` fields description](/statement-summary-tables.md#statements_summary-fields-description). + ## Monitoring metrics and charts @@ -508,7 +603,7 @@ You can view the data of resource groups in the current [`RESOURCE_GROUPS`](/inf > **Note:** > -> This section is only applicable to TiDB Self-Hosted. Currently, TiDB Cloud does not provide resource control metrics. +> This section is only applicable to TiDB Self-Managed. Currently, TiDB Cloud does not provide resource control metrics. TiDB regularly collects runtime information about resource control and provides visual charts of the metrics in Grafana's **TiDB** > **Resource Control** dashboard. @@ -539,4 +634,4 @@ The resource control feature does not impact the regular usage of data import, e * [CREATE RESOURCE GROUP](/sql-statements/sql-statement-create-resource-group.md) * [ALTER RESOURCE GROUP](/sql-statements/sql-statement-alter-resource-group.md) * [DROP RESOURCE GROUP](/sql-statements/sql-statement-drop-resource-group.md) -* [RESOURCE GROUP RFC](https://github.com/pingcap/tidb/blob/master/docs/design/2022-11-25-global-resource-control.md) +* [RESOURCE GROUP RFC](https://github.com/pingcap/tidb/blob/release-7.5/docs/design/2022-11-25-global-resource-control.md) diff --git a/tidb-roadmap.md b/tidb-roadmap.md deleted file mode 100644 index 0b7abfdc6ae26..0000000000000 --- a/tidb-roadmap.md +++ /dev/null @@ -1,236 +0,0 @@ ---- -title: TiDB Roadmap -summary: Learn about what's coming in the future for TiDB. ---- - -# TiDB Roadmap - -This roadmap provides a look into the proposed future. This will be continually updated as we release long-term stable (LTS) versions. The purpose is to provide visibility into what is coming, so that you can more closely follow the progress, learn about the key milestones on the way, and give feedback as the development work goes on. - -In the course of development, this roadmap is subject to change based on user needs and feedback. As expected, as the columns move right, the items under them are less committed. If you have a feature request or want to prioritize a feature, please file an issue on [GitHub](https://github.com/pingcap/tidb/issues). - -## Rolling roadmap highlights - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    CategoryEnd of CY23 LTS releaseMid of CY24 LTS releaseFuture releases
    - Scalability and Performance
    Enhance horsepower -
    -
      -
    • - Distributed execution framework
      - In v7.2.0, TiDB introduced the distributed execution framework for background tasks (such as DDL and analyze). This is the foundation for parallelizing these tasks across compute nodes. v7.4.0 introduces global sorting in distributed re-organization tasks (such as DDL and import), which greatly mitigates extra resource consumption in storage. Optionally, external shared storage can be leveraged for simplicity and cost savings. -
    • -
      -
      -
    -
    -
      -
    • - Enhancements to performance and generalizability of plan cache
      -
    • -
      -
    • - Dynamic node scaling via distributed execution framework
      - Automatically adjust node allocation to meet resource costs of background tasks, while maintaining stability and performance expectations -
    • -
      -
      -
    -
    -
      -
    • - Unlimited transaction size -
    • -
      -
    • - Federated query -
      TiDB query planner support for multiple storage engines in HTAP use cases. -
    • -
      -
    -
    - Reliability and Availability -
    Enhance dependability -
    -
      -
    • - Resource control for background tasks
      - - Control over how background tasks, such as imports, DDL, TTL, auto-analyze, and compactions, can affect foreground traffic - -
    • -
      -
    • - Runaway query control
      - An operator-controlled way to greatly enhance performance stability for workloads with unexpectedly expensive queries - -
    • -
    -
    -
      -
    • - Disaggregation of Placement Driver (PD) -
      Enhance cluster scalability and resilience -
    • -
      -
    -
    -
      -
    • - Multi-tenancy -
      Resource isolation on top of resource control -
    • -
      -
    -
    - Database Operations and Observability -
    Enhance DB manageability and its ecosystem -
    -
      -
    • - TiCDC integrations with data warehouse or data lake systems -
      -
    • -
      -
    • - TiDB node labels -
      Assign existing or newly added TiDB nodes for DDL operations to isolate DDL tasks from the compute resources used by online traffic - -
    • -
      -
    -
    -
      -
    • - SQL plan management -
      Mechanism for controlling SQL plan regression -
    • -
      -
    • - Index Advisor -
      Offer index recommendations to users based on workload, statistics, and execution plans -
    • -
    -
    -
      -
    • - Materialized views -
      Store pre-computed results as a persistent data view to boost query performance -
    • -
      -
    • - Heterogeneous database migration support -
    • -
      -
    -
    - Security -
    Enhance data safety and privacy -
    -
      -
    • - Key management via Azure Key Vault -
      Static encryption managed by Azure Key Vault -
    • -
      -
    • - Column-level access control -
      Grant and restrict access to specific columns -
    • -
      -
    -
    -
      -
    • - IAM authentication for AWS -
      TiDB as AWS third-party ARN for AWS IAM access -
    • -
      -
    • - Unified TLS CA/Key rotation policy -
      Unified certificate management mechanism for all TiDB components -
    • -

      -
    • - AWS FIPS support -
      Enable FedRAMP compliance -
    • -
    -
    -
      -
    • - Label-based access control -
      Access permissions granted by configured labels -
    • -
      -
    • - Enhanced client-side encryption -
    • -
      -
    • - Enhanced data masking -
    • -
      -
    -
    - -These are non-exhaustive plans and are subject to change. Features might differ per service subscriptions. - -## Previously delivered roadmap items - -You might have been waiting on some items from the last version. The following lists some previously delivered features. For more details, refer to the [v7.1.0 release notes](/releases/release-7.1.0.md). - -- Foundation of multi-tenancy framework: resource control quotas and scheduling for resource groups -- TiCDC supports object storage sink, including Amazon S3 and Azure Blob Storage (GA) -- Fastest online `ADD INDEX` (GA) -- TiFlash late materialization (GA) -- TiFlash supports spill to disk (GA) -- LDAP authentication -- SQL audit log enhancement (Enterprise-only) -- Partitioned Raft KV storage engine (experimental) -- General session-level plan cache (experimental) -- TiCDC distributed per table with Kafka downstream (experimental) - -## Recently shipped - -- [TiDB 7.4.0 Release Notes](https://docs.pingcap.com/tidb/v7.4/release-7.4.0) -- [TiDB 7.3.0 Release Notes](https://docs.pingcap.com/tidb/v7.3/release-7.3.0) -- [TiDB 7.2.0 Release Notes](https://docs.pingcap.com/tidb/v7.2/release-7.2.0) -- [TiDB 7.1.0 Release Notes](https://docs.pingcap.com/tidb/v7.1/release-7.1.0) -- [TiDB 7.0.0 Release Notes](https://docs.pingcap.com/tidb/v7.0/release-7.0.0) -- [TiDB 6.6.0 Release Notes](https://docs.pingcap.com/tidb/v6.6/release-6.6.0) -- [TiDB 6.5.0 Release Notes](https://docs.pingcap.com/tidb/v6.5/release-6.5.0) diff --git a/tidb-rowid.md b/tidb-rowid.md new file mode 100644 index 0000000000000..48a30eec7ee45 --- /dev/null +++ b/tidb-rowid.md @@ -0,0 +1,157 @@ +--- +title: _tidb_rowid +summary: Learn what `_tidb_rowid` is, when it is available, and how to use it safely. +--- + +# `_tidb_rowid` + +`_tidb_rowid` is a hidden system column automatically generated by TiDB. For tables that do not use a clustered index, it serves as the internal row ID of the table. You cannot declare or modify this column in the table schema, but you can reference it in SQL when the table uses `_tidb_rowid` as its internal row ID. + +In the current implementation, `_tidb_rowid` is an extra `BIGINT NOT NULL` column automatically managed by TiDB. + +> **Warning:** +> +> - Do not assume that `_tidb_rowid` is globally unique in all cases. For partitioned tables that do not use clustered indexes, executing `ALTER TABLE ... EXCHANGE PARTITION` might result in duplicate `_tidb_rowid` values across different partitions. +> - If you need a stable unique identifier, define and use an explicit primary key instead of relying on `_tidb_rowid`. + +## When `_tidb_rowid` is available + +TiDB uses `_tidb_rowid` to identify each row when a table does not use a clustered primary key as the unique row identifier. In practice, this means that the following types of tables use `_tidb_rowid`: + +- Tables without primary keys +- Tables with primary keys that are explicitly defined as `NONCLUSTERED` + +`_tidb_rowid` is not available for tables that use a clustered index (that is, tables whose primary key is defined as `CLUSTERED`, regardless of whether it is a single-column or composite primary key). + +The following example shows the difference: + +```sql +CREATE TABLE t1 (a INT, b VARCHAR(20)); +CREATE TABLE t2 (id BIGINT PRIMARY KEY NONCLUSTERED, a INT); +CREATE TABLE t3 (id BIGINT PRIMARY KEY CLUSTERED, a INT); +``` + +For `t1` and `t2`, you can query `_tidb_rowid` because these tables do not use a clustered index as the row identifier: + +```sql +SELECT _tidb_rowid, a, b FROM t1; +SELECT _tidb_rowid, id, a FROM t2; +``` + +For `t3`, `_tidb_rowid` is unavailable because the clustered primary key is already the row identifier: + +```sql +SELECT _tidb_rowid, id, a FROM t3; +``` + +```sql +ERROR 1054 (42S22): Unknown column '_tidb_rowid' in 'field list' +``` + +## Read `_tidb_rowid` + +For tables that use `_tidb_rowid`, you can query `_tidb_rowid` in `SELECT` statements. This is useful for tasks such as pagination, troubleshooting, and batch processing. + +Example: + +```sql +CREATE TABLE t (a INT, b VARCHAR(20)); +INSERT INTO t VALUES (1, 'x'), (2, 'y'); + +SELECT _tidb_rowid, a, b FROM t ORDER BY _tidb_rowid; +``` + +```sql ++-------------+---+---+ +| _tidb_rowid | a | b | ++-------------+---+---+ +| 1 | 1 | x | +| 2 | 2 | y | ++-------------+---+---+ +``` + +To view the next value that TiDB will allocate for the row ID, use `SHOW TABLE ... NEXT_ROW_ID`: + +```sql +SHOW TABLE t NEXT_ROW_ID; +``` + +```sql ++-----------------------+------------+-------------+--------------------+-------------+ +| DB_NAME | TABLE_NAME | COLUMN_NAME | NEXT_GLOBAL_ROW_ID | ID_TYPE | ++-----------------------+------------+-------------+--------------------+-------------+ +| update_doc_rowid_test | t | _tidb_rowid | 30001 | _TIDB_ROWID | ++-----------------------+------------+-------------+--------------------+-------------+ +``` + +## Write `_tidb_rowid` + +By default, TiDB does not allow `INSERT`, `REPLACE`, or `UPDATE` statements to write `_tidb_rowid` directly. + +```sql +INSERT INTO t(_tidb_rowid, a, b) VALUES (101, 4, 'w'); +``` + +```sql +ERROR 1105 (HY000): insert, update and replace statements for _tidb_rowid are not supported +``` + +If you need to preserve the original row IDs during data import or migration, enable the [`tidb_opt_write_row_id`](/system-variables.md#tidb_opt_write_row_id) system variable first: + +```sql +SET @@tidb_opt_write_row_id = ON; +INSERT INTO t(_tidb_rowid, a, b) VALUES (100, 3, 'z'); +SET @@tidb_opt_write_row_id = OFF; + +SELECT _tidb_rowid, a, b FROM t WHERE _tidb_rowid = 100; +``` + +```sql ++-------------+---+---+ +| _tidb_rowid | a | b | ++-------------+---+---+ +| 100 | 3 | z | ++-------------+---+---+ +``` + +> **Warning:** +> +> `tidb_opt_write_row_id` is intended for import and migration scenarios. It is not recommended for regular application writes. + +## Restrictions + +- You cannot create a user column named `_tidb_rowid`. +- You cannot rename an existing user column to `_tidb_rowid`. +- `_tidb_rowid` is an internal column in TiDB. Do not treat it as a business primary key or a long-term identifier. +- On partitioned non-clustered tables, `_tidb_rowid` values are not guaranteed to be unique across partitions. After you execute `EXCHANGE PARTITION`, different partitions can contain rows with the same `_tidb_rowid` value. +- Whether `_tidb_rowid` exists depends on the table schema. For tables with clustered indexes, use the primary key as the row identifier. + +## Address hotspot issues + +For tables that use `_tidb_rowid`, TiDB allocates row IDs in increasing order by default. In write-intensive workloads, this can create write hotspots. + +To mitigate this issue (for tables that rely on `_tidb_rowid` as the row ID), consider using [`SHARD_ROW_ID_BITS`](/shard-row-id-bits.md) to distribute row IDs more evenly, and use [`PRE_SPLIT_REGIONS`](/sql-statements/sql-statement-split-region.md#pre_split_regions) to pre-split Regions when necessary. + +Example: + +```sql +CREATE TABLE t ( + id BIGINT PRIMARY KEY NONCLUSTERED, + c INT +) SHARD_ROW_ID_BITS = 4; +``` + +`SHARD_ROW_ID_BITS` applies only to tables that use `_tidb_rowid` and does not apply to tables with clustered indexes. + +## Related statements and variables + +- [`SHOW TABLE NEXT_ROW_ID`](/sql-statements/sql-statement-show-table-next-rowid.md): shows the next row ID that TiDB will allocate +- [`SHARD_ROW_ID_BITS`](/shard-row-id-bits.md): shards implicit row IDs to reduce hotspots +- [`Clustered Indexes`](/clustered-indexes.md): explains when a table uses the primary key instead of `_tidb_rowid` +- [`tidb_opt_write_row_id`](/system-variables.md#tidb_opt_write_row_id): controls whether writes to `_tidb_rowid` are allowed + +## See also + +- [`CREATE TABLE`](/sql-statements/sql-statement-create-table.md) +- [`AUTO_INCREMENT`](/auto-increment.md) +- [Non-transactional DML](/non-transactional-dml.md) diff --git a/tidb-scheduling.md b/tidb-scheduling.md index 2cfe2a43fa0e3..737ac2b18bcd6 100644 --- a/tidb-scheduling.md +++ b/tidb-scheduling.md @@ -67,7 +67,7 @@ Scheduling is based on information collection. In short, the PD scheduling compo - State information reported by each TiKV peer: - Each TiKV peer sends heartbeats to PD periodically. PD not only checks whether the store is alive, but also collects [`StoreState`](https://github.com/pingcap/kvproto/blob/master/proto/pdpb.proto#L473) in the heartbeat message. `StoreState` includes: + Each TiKV peer sends heartbeats to PD periodically. PD not only checks whether the store is alive, but also collects [`StoreState`](https://github.com/pingcap/kvproto/blob/release-7.5/proto/pdpb.proto#L473) in the heartbeat message. `StoreState` includes: * Total disk space * Available disk space @@ -83,13 +83,13 @@ Scheduling is based on information collection. In short, the PD scheduling compo + **Disconnect**: Heartbeat messages between the PD and the TiKV store are lost for more than 20 seconds. If the lost period exceeds the time specified by `max-store-down-time`, the status "Disconnect" changes to "Down". + **Down**: Heartbeat messages between the PD and the TiKV store are lost for a time longer than `max-store-down-time` (30 minutes by default). In this status, the TiKV store starts replenishing replicas of each Region on the surviving store. + **Offline**: A TiKV store is manually taken offline through PD Control. This is only an intermediate status for the store to go offline. The store in this status moves all its Regions to other "Up" stores that meet the relocation conditions. When `leader_count` and `region_count` (obtained through PD Control) both show `0`, the store status changes to "Tombstone" from "Offline". In the "Offline" status, **do not** disable the store service or the physical server where the store is located. During the process that the store goes offline, if the cluster does not have target stores to relocate the Regions (for example, inadequate stores to hold replicas in the cluster), the store is always in the "Offline" status. - + **Tombstone**: The TiKV store is completely offline. You can use the `remove-tombstone` interface to safely clean up TiKV in this status. + + **Tombstone**: The TiKV store is completely offline. You can use the `remove-tombstone` interface to safely clean up TiKV in this status. Starting from v6.5.0, if not manually handled, PD will automatically delete the Tombstone records stored internally one month after the node is converted to Tombstone. ![TiKV store status relationship](/media/tikv-store-status-relationship.png) - Information reported by Region leaders: - Each Region leader sends heartbeats to PD periodically to report [`RegionState`](https://github.com/pingcap/kvproto/blob/master/proto/pdpb.proto#L312), including: + Each Region leader sends heartbeats to PD periodically to report [`RegionState`](https://github.com/pingcap/kvproto/blob/release-7.5/proto/pdpb.proto#L312), including: * Position of the leader itself * Positions of other replicas diff --git a/tidb-storage.md b/tidb-storage.md index ee2c0115f257f..624e0d941409e 100644 --- a/tidb-storage.md +++ b/tidb-storage.md @@ -74,7 +74,9 @@ As we distribute and replicate data in Regions, we have a distributed Key-Value ## MVCC -Many databases implement multi-version concurrency control (MVCC), and TiKV is no exception. Imagine the situation where two clients modify the value of a Key at the same time. Without MVCC, the data needs to be locked. In a distributed scenario, it might cause performance and deadlock problems. TiKV's MVCC implementation is achieved by appending a version number to Key. In short, without MVCC, TiKV's data layout can be seen as: +TiKV supports multi-version concurrency control (MVCC). Consider a scenario where Client A is writing to a key simultaneously as Client B is reading the same key. Without the MVCC mechanism, these read and write operations would be mutually exclusive, posing performance issues and deadlocks in a distributed scenario. However, With MVCC, as long as Client B performs a read operation at a logical time earlier than the Client A write operation, then Client B can correctly read the original value at the same time Client A performs the write operation. Even if the key is modified multiple times by multiple write operations, Client B can still read the old value according to its logical time. + +TiKV MVCC is implemented by appending a version number to the key. Without MVCC, the Key-Value pairs of TiKV are as follows: ``` Key1 -> Value @@ -83,7 +85,7 @@ Key2 -> Value KeyN -> Value ``` -With MVCC, the key array of TiKV is like this: +With MVCC, the Key-Value pairs of TiKV are as follows: ``` Key1_Version3 -> Value @@ -104,4 +106,4 @@ Note that for multiple versions of the same Key, versions with larger numbers ar ## Distributed ACID transaction -Transaction of TiKV adopts the model used by Google in BigTable: [Percolator](https://research.google.com/pubs/pub36726.html). TiKV's implementation is inspired by this paper, with a lot of optimizations. See [transaction overview](/transaction-overview.md) for details. +Transaction of TiKV adopts the model used by Google in BigTable: [Percolator](https://research.google/pubs/large-scale-incremental-processing-using-distributed-transactions-and-notifications/). TiKV's implementation is inspired by this paper, with a lot of optimizations. See [transaction overview](/transaction-overview.md) for details. diff --git a/tidb-troubleshooting-map.md b/tidb-troubleshooting-map.md index a5a3a5dadb6d5..026ba7851b370 100644 --- a/tidb-troubleshooting-map.md +++ b/tidb-troubleshooting-map.md @@ -1,7 +1,6 @@ --- title: TiDB Troubleshooting Map summary: Learn how to troubleshoot common errors in TiDB. -aliases: ['/docs/dev/tidb-troubleshooting-map/','/docs/dev/how-to/troubleshoot/diagnose-map/'] --- # TiDB Troubleshooting Map @@ -59,11 +58,13 @@ Refer to [5 PD issues](#5-pd-issues). - 3.1.2 TiDB DDL job hangs or executes slowly (use `admin show ddl jobs` to check DDL progress) - - Cause 1: Network issue with other components (PD/TiKV). + - Cause 1: TiDB introduces the [metadata lock](/metadata-lock.md) in v6.3.0, and enables it by default in v6.5.0 and later versions. If the table involved in the DDL operation has intersections with the table involved in the uncommitted transaction, the DDL operation is blocked until the transaction is committed or rolled back. - - Cause 2: Early versions of TiDB (earlier than v3.0.8) have heavy internal load because of a lot of goroutine at high concurrency. + - Cause 2: Network issue with other components (PD/TiKV). - - Cause 3: In early versions (v2.1.15 & versions < v3.0.0-rc1), PD instances fail to delete TiDB keys, which causes every DDL change to wait for two leases. + - Cause 3: Early versions of TiDB (earlier than v3.0.8) have heavy internal load because of a lot of goroutine at high concurrency. + + - Cause 4: In early versions (v2.1.15 & versions < v3.0.0-rc1), PD instances fail to delete TiDB keys, which causes every DDL change to wait for two leases. - For other unknown causes, [report a bug](https://github.com/pingcap/tidb/issues/new?labels=type%2Fbug&template=bug-report.md). @@ -241,15 +242,13 @@ Check the specific cause for busy by viewing the monitor **Grafana** -> **TiKV** - 4.3.1 TiKV RocksDB encounters `write stall`. - A TiKV instance has two RocksDB instances, one in `data/raft` to save the Raft log, another in `data/db` to save the real data. You can check the specific cause for stall by running `grep "Stalling" RocksDB` in the log. The RocksDB log is a file starting with `LOG`, and `LOG` is the current log. - - - Too many `level0 sst` causes stall. You can add the `[rocksdb] max-sub-compactions = 2` (or 3) parameter to speed up `level0 sst` compaction. The compaction task from level0 to level1 is divided into several subtasks (the max number of subtasks is the value of `max-sub-compactions`) to be executed concurrently. See [case-815](https://github.com/pingcap/tidb-map/blob/master/maps/diagnose-case-study/case815.md) in Chinese. + A TiKV instance has two RocksDB instances, one in `data/raft` to store the Raft log, and the other in `data/db` to store the real data. You can check the specific cause for the stall by running `grep "Stalling" RocksDB` in the log. The RocksDB log is a file starting with `LOG`, and `LOG` is the current log. `write stall` is a performance degradation mechanism natively built in RocksDB. When `write stall` occurs in RocksDB, the system performance drops significantly. In versions before v5.2.0, TiDB tries to block all write requests by returning a `ServerIsBusy` error directly to the client when encountering `write stall`, but this could lead to a sharp decrease in QPS performance. Starting from v5.2.0, TiKV introduces a new flow control mechanism that suppresses writes by delaying write requests dynamically in the scheduling layer, which replaces the previous mechanism of returning `server is busy` to clients when `write stall` occurs. The new flow control mechanism is enabled by default, and TiKV automatically disables the `write stall` mechanism for `KvDB` and `RaftDB` (except for memtables). However, when the number of pending requests exceeds a certain threshold, the flow control mechanism still takes effect, beginning to reject some or all write requests and returning a `server is busy` error to clients. For detailed explanations and thresholds, see [Flow control configuration](/tikv-configuration-file.md#storageflow-control). - - Too many `pending compaction bytes` causes stall. The disk I/O fails to keep up with the write operations in business peaks. You can mitigate this problem by increasing the `soft-pending-compaction-bytes-limit` and `hard-pending-compaction-bytes-limit` of the corresponding CF. + - If the `server is busy` error is triggered by too many pending compaction bytes, you can alleviate this issue by increasing the values of the [`soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#soft-pending-compaction-bytes-limit) and [`hard-pending-compaction-bytes-limit`](/tikv-configuration-file.md#hard-pending-compaction-bytes-limit) parameters. - - The default value of `[rocksdb.defaultcf] soft-pending-compaction-bytes-limit` is `64GB`. If the pending compaction bytes reaches the threshold, RocksDB slows down the write speed. You can set `[rocksdb.defaultcf] soft-pending-compaction-bytes-limit` to `128GB`. + - When pending compaction bytes reach the value of the `soft-pending-compaction-bytes-limit` parameter (`192GiB` by default), the flow control mechanism begins to reject some write requests (by returning `ServerIsBusy` to clients). In this case, you can increase the value of this parameter, for example, `[storage.flow-control] soft-pending-compaction-bytes-limit = "384GiB"`. - - The default value of `hard-pending-compaction-bytes-limit` is `256GB`. If the pending compaction bytes reaches the threshold (this is not likely to happen, because RocksDB slows down the write after the pending compaction bytes reaches `soft-pending-compaction-bytes-limit`), RocksDB stops the write operation. You can set `hard-pending-compaction-bytes-limit` to `512GB`. + - When pending compaction bytes reach the value of the `hard-pending-compaction-bytes-limit` parameter (`1024GiB` by default), the flow control mechanism begins to reject all write requests (by returning `ServerIsBusy` to clients). This scenario is less likely to occur because the flow control mechanism intervenes to slow down write speed after reaching the threshold of `soft-pending-compaction-bytes-limit`. If it occurs, you can increase the value of this parameter, for example, `[storage.flow-control] hard-pending-compaction-bytes-limit = "2048GiB"`. - If the disk I/O capacity fails to keep up with the write for a long time, it is recommended to scale up your disk. If the disk throughput reaches the upper limit and causes write stall (for example, the SATA SSD is much lower than NVME SSD), while the CPU resources is sufficient, you may apply a compression algorithm of higher compression ratio. This way, the CPU resources is traded for disk resources, and the pressure on the disk is eased. @@ -414,7 +413,7 @@ Check the specific cause for busy by viewing the monitor **Grafana** -> **TiKV** - The binlog data is too large, so the single message written to Kafka is too large. You need to modify the following configuration of Kafka: - ```conf + ```properties message.max.bytes=1073741824 replica.fetch.max.bytes=1073741824 fetch.message.max.bytes=1073741824 @@ -470,13 +469,13 @@ Check the specific cause for busy by viewing the monitor **Grafana** -> **TiKV** ### 6.2 Data Migration -- 6.2.1 TiDB Data Migration (DM) is a migration tool that supports data migration from MySQL/MariaDB into TiDB. For details, see [DM on GitHub](https://github.com/pingcap/dm/). +- 6.2.1 TiDB Data Migration (DM) is a migration tool that supports data migration from MySQL/MariaDB into TiDB. For details, see [DM overview](/dm/dm-overview.md). - 6.2.2 `Access denied for user 'root'@'172.31.43.27' (using password: YES)` shows when you run `query status` or check the log. - The database related passwords in all the DM configuration files should be encrypted by `dmctl`. If a database password is empty, it is unnecessary to encrypt the password. Cleartext passwords can be used since v1.0.6. - During DM operation, the user of the upstream and downstream databases must have the corresponding read and write privileges. Data Migration also [prechecks the corresponding privileges](/dm/dm-precheck.md) automatically while starting the data replication task. - - To deploy different versions of DM-worker/DM-master/dmctl in a DM cluster, see the [case study on AskTUG](https://asktug.com/t/dm1-0-0-ga-access-denied-for-user/1049/5) in Chinese. + - To deploy different versions of DM-worker/DM-master/dmctl in a DM cluster, see the [case study on AskTUG](https://pingkai.cn/tidbcommunity/forum/t/topic/1049/5) in Chinese. - 6.2.3 A replication task is interrupted with the `driver: bad connection` error returned. @@ -518,7 +517,7 @@ Check the specific cause for busy by viewing the monitor **Grafana** -> **TiKV** ### 6.3 TiDB Lightning -- 6.3.1 TiDB Lightning is a tool for fast full import of large amounts of data into a TiDB cluster. See [TiDB Lightning on GitHub](https://github.com/pingcap/tidb/tree/master/br/pkg/lightning). +- 6.3.1 TiDB Lightning is a tool for fast full import of large amounts of data into a TiDB cluster. See [TiDB Lightning on GitHub](https://github.com/pingcap/tidb/tree/release-7.5/br/pkg/lightning). - 6.3.2 Import speed is too slow. @@ -572,19 +571,13 @@ Check the specific cause for busy by viewing the monitor **Grafana** -> **TiKV** You can increase the GC lifetime by modifying the [`tidb_gc_life_time`](/system-variables.md#tidb_gc_life_time-new-in-v50) system variable. Generally, it is not recommended to modify this parameter, because changing it might cause many old versions to pile up if this transaction has a large number of `UPDATE` and `DELETE` statements. -- 7.1.2 `txn takes too much time`. - - This error is returned when you commit a transaction that has not been committed for a long time (over 590 seconds). - - If your application needs to execute a transaction of such a long time, you can increase the `[tikv-client] max-txn-time-use = 590` parameter and the GC lifetime to avoid this issue. It is recommended to check whether your application needs such a long transaction time. - -- 7.1.3 `coprocessor.go` reports `request outdated`. +- 7.1.2 `coprocessor.go` reports `request outdated`. This error is returned when the coprocessor request sent to TiKV waits in a queue at TiKV for over 60 seconds. You need to investigate why the TiKV coprocessor is in a long queue. -- 7.1.4 `region_cache.go` reports a large number of `switch region peer to next due to send request fail`, and the error message is `context deadline exceeded`. +- 7.1.3 `region_cache.go` reports a large number of `switch region peer to next due to send request fail`, and the error message is `context deadline exceeded`. The request for TiKV timed out and triggers the region cache to switch the request to other nodes. You can continue to run the `grep " cancelled` command on the `addr` field in the log and take the following steps according to the `grep` results: @@ -595,7 +588,7 @@ Check the specific cause for busy by viewing the monitor **Grafana** -> **TiKV** - `wait response is cancelled`: The request timed out after it is sent to TiKV. You need to check the response time of the corresponding TiKV address and the Region logs in PD and KV at that time. -- 7.1.5 `distsql.go` reports `inconsistent index`. +- 7.1.4 `distsql.go` reports `inconsistent index`. The data index seems to be inconsistent. Run the `admin check table ` command on the table where the reported index is. If the check fails, disable garbage collection by running the following command, and [report a bug](https://github.com/pingcap/tidb/issues/new?labels=type%2Fbug&template=bug-report.md): diff --git a/tidb-upgrade-migration-guide.md b/tidb-upgrade-migration-guide.md new file mode 100644 index 0000000000000..7671ed11a336b --- /dev/null +++ b/tidb-upgrade-migration-guide.md @@ -0,0 +1,327 @@ +--- +title: Migrate and Upgrade a TiDB Cluster +summary: Learn how to migrate and upgrade a TiDB cluster using BR for full backup and restore, along with TiCDC for incremental data replication. +--- + +# Migrate and Upgrade a TiDB Cluster + +This document describes how to migrate and upgrade a TiDB cluster (also known as a blue-green upgrade) using [BR](/br/backup-and-restore-overview.md) for full backup and restore, along with [TiCDC](/ticdc/ticdc-overview.md) for incremental data replication. This solution uses dual-cluster redundancy and incremental replication to enable smooth traffic switchover and fast rollback, providing a reliable and low-risk upgrade path for critical systems. It is recommended to regularly upgrade the database version to continuously benefit from performance improvements and new features, helping you maintain a secure and efficient database system. The key advantages of this solution include: + +- **Controllable risk**: supports rollback to the original cluster within minutes, ensuring business continuity. +- **Data integrity**: uses a multi-stage verification mechanism to prevent data loss. +- **Minimal business impact**: requires only a brief maintenance window for the final switchover. + +The core workflow for migration and upgrade is as follows: + +1. **Pre-check risks**: verify cluster status and solution feasibility. +2. **Prepare the new cluster**: create a new cluster from a full backup of the old cluster and upgrade it to the target version. +3. **Replicate incremental data**: establish a forward data replication channel using TiCDC. +4. **Switch and verify**: perform multi-dimensional verification, switch business traffic to the new cluster, and set up a TiCDC reverse replication channel. +5. **Observe status**: maintain the reverse replication channel. After the observation period, clean up the environment. + +**Rollback plan**: if the new cluster encounters issues during the migration and upgrade process, you can switch business traffic back to the original cluster at any time. + +The following sections describe the standardized process and general steps for migrating and upgrading a TiDB cluster. The example commands are based on a TiDB Self-Managed environment. + +## Step 1: Evaluate solution feasibility + +Before migrating and upgrading, evaluate the compatibility of relevant components and check cluster health status. + +- Check the TiDB cluster version: this solution applies to TiDB v6.5.0 or later versions. + +- Verify TiCDC compatibility: + + - **Table schema requirements**: ensure that tables to be replicated contain valid indexes. For more information, see [TiCDC valid index](/ticdc/ticdc-overview.md#best-practices). + - **Feature limitations**: TiCDC does not support Sequence or TiFlash DDL replication. For more information, see [TiCDC unsupported scenarios](/ticdc/ticdc-overview.md#unsupported-scenarios). + - **Best practices**: avoid executing DDL operations on the upstream cluster of TiCDC during switchover. + +- Verify BR compatibility: + + - Review the compatibility matrix of BR full backup. For more information, see [BR version compatibility matrix](/br/backup-and-restore-overview.md#version-compatibility). + - Check the known limitations of BR backup and restore. For more information, see [BR usage restrictions](/br/backup-and-restore-overview.md#restrictions). + +- Check the health status of the cluster, such as [Region](/glossary.md#regionpeerraft-group) health and node resource utilization. + +## Step 2: Prepare the new cluster + +### 1. Adjust the GC lifetime of the old cluster + +To ensure data replication stability, adjust the system variable [`tidb_gc_life_time`](/system-variables.md#tidb_gc_life_time-new-in-v50) to a value that covers the total duration of the following operations and intervals: BR backup, BR restore, cluster upgrade, and TiCDC Changefeed replication setup. Otherwise, the replication task might enter an unrecoverable `failed` state, requiring a restart of the entire migration and upgrade process from a new full backup. + +The following example sets `tidb_gc_life_time` to `60h`: + +```sql +-- Check the current GC lifetime setting. +SHOW VARIABLES LIKE '%tidb_gc_life_time%'; +-- Set GC lifetime. +SET GLOBAL tidb_gc_life_time=60h; +``` + +> **Note:** +> +> Increasing `tidb_gc_life_time` increases storage usage for [MVCC](/glossary.md#multi-version-concurrency-control-mvcc) data and might affect query performance. For more information, see [GC Overview](/garbage-collection-overview.md). Adjust the GC duration based on estimated operation time while considering storage and performance impacts. + +### 2. Migrate full data to the new cluster + +When migrating full data to the new cluster, note the following: + +- **Version compatibility**: the BR version used for backup and restore must match the major version of the old cluster. +- **Performance impact**: BR backup consumes system resources. To minimize business impact, perform backups during off-peak hours. +- **Time estimation**: under optimal hardware conditions (no disk I/O or network bandwidth bottlenecks), estimated times are: + + - Backup speed: backing up 1 TiB of data per TiKV node with 8 threads takes approximately 1 hour. + - Restore speed: restoring 1 TiB of data per TiKV node takes approximately 20 minutes. + +- **Configuration consistency**: ensure that the [`new_collations_enabled_on_first_bootstrap`](/tidb-configuration-file.md#new_collations_enabled_on_first_bootstrap) configuration is identical between the old and new clusters. Otherwise, BR restore will fail. +- **System table restore**: Use the `--with-sys-table` option during BR restore to recover system table data. + +To migrate full data to the new cluster, take the following steps: + +1. Perform a full backup on the old cluster: + + ```shell + tiup br:${cluster_version} backup full --pd ${pd_host}:${pd_port} -s ${backup_location} + ``` + +2. Record the TSO of the old cluster for later TiCDC Changefeed creation: + + ```shell + tiup br:${cluster_version} validate decode --field="end-version" \ + --storage "s3://xxx?access-key=${access-key}&secret-access-key=${secret-access-key}" | tail -n1 + ``` + +3. Deploy the new cluster: + + ```shell + tiup cluster deploy ${new_cluster_name} ${cluster_version} tidb-cluster.yaml + ``` + +4. Restore the full backup to the new cluster: + + ```shell + tiup br:${cluster_version} restore full --pd ${pd_host}:${pd_port} -s ${backup_location} --with-sys-table + ``` + +### 3. Upgrade the new cluster to the target version + +To save time, you can perform an offline upgrade using the following commands. For more upgrade methods, see [Upgrade TiDB Using TiUP](/upgrade-tidb-using-tiup.md). + +```shell +tiup cluster stop # Stop the cluster +tiup cluster upgrade --offline # Perform offline upgrade +tiup cluster start # Start the cluster +``` + +To maintain business continuity, you need to replicate essential configurations from the old cluster to the new cluster, such as configuration items and system variables. + +## Step 3: Replicate incremental data + +### 1. Establish a forward data replication channel + +At this stage, the old cluster remains at its original version, while the new cluster has been upgraded to the target version. In this step, you need to establish a forward data replication channel from the old cluster to the new cluster. + +> **Note:** +> +> The TiCDC component version must match the major version of the old cluster. + +- Create a Changefeed task and set the incremental replication starting point (`${tso}`) to the exact backup TSO recorded in [Step 2](#step-2-prepare-the-new-cluster) to prevent data loss: + + ```shell + tiup ctl:${cluster_version} cdc changefeed create --server http://${cdc_host}:${cdc_port} --sink-uri="mysql://${username}:${password}@${tidb_endpoint}:${port}" --config config.toml --start-ts ${tso} + ``` + +- Check the replication task status and confirm that `tso` or `checkpoint` is continuously advancing: + + ```shell + tiup ctl:${cluster_version} cdc changefeed list --server http://${cdc_host}:${cdc_port} + ``` + + The output is as follows: + + ```shell + [{ + "id": "cdcdb-cdc-task-standby", + "summary": { + "state": "normal", + "tso": 417886179132964865, + "checkpoint": "202x-xx-xx xx:xx:xx.xxx", + "error": null + } + }] + ``` + +During incremental data replication, continuously monitor the replication channel status and adjust settings if needed: + +- Latency metrics: ensure that `Changefeed checkpoint lag` remains within an acceptable range, such as within 5 minutes. +- Throughput health: ensure that `Sink flush rows/s` consistently exceeds the business write rate. +- Errors and alerts: regularly check TiCDC logs and alert information. +- (Optional) Test data replication: update test data and verify that Changefeed correctly replicates it to the new cluster. +- (Optional) Adjust the TiCDC configuration item [`gc-ttl`](/ticdc/ticdc-server-config.md) (defaults to 24 hours). + + If a replication task is unavailable or interrupted and cannot be resolved in time, `gc-ttl` ensures that data needed by TiCDC is retained in TiKV without being cleaned by garbage collection (GC). If this duration is exceeded, the replication task enters a `failed` state and cannot recover. In this case, PD's GC safe point continues advancing, requiring a new backup to restart the process. + + Increasing the value of `gc-ttl` accumulates more MVCC data, similar to increasing `tidb_gc_life_time`. It is recommended to set it to a reasonably long but appropriate value. + +### 2. Verify data consistency + +After data replication is complete, verify data consistency between the old and new clusters using the following methods: + +- Use the [sync-diff-inspector](/sync-diff-inspector/sync-diff-inspector-overview.md) tool: + + ```shell + ./sync_diff_inspector --config=./config.toml + ``` + +- Use the snapshot configuration of [sync-diff-inspector](/sync-diff-inspector/sync-diff-inspector-overview.md) with the [Syncpoint](/ticdc/ticdc-upstream-downstream-check.md) feature of TiCDC to verify data consistency without stopping Changefeed replication. For more information, see [Upstream and Downstream Clusters Data Validation and Snapshot Read](/ticdc/ticdc-upstream-downstream-check.md). + +- Perform manual validation of business data, such as comparing table row counts. + +### 3. Finalize the environment setup + +This migration procedure restores some system table data using the BR `--with-sys-table` option. For tables that are not included in the scope, you need to manually restore. Common items to check and supplement include: + +- User privileges: compare the `mysql.user` table. +- Configuration settings: ensure that configuration items and system variables are consistent. +- Auto-increment columns: clear auto-increment ID caches in the new cluster. +- Statistics: collect statistics manually or enable automatic collection in the new cluster. + +Additionally, you can scale out the new cluster to handle expected workloads and migrate operational tasks, such as alert subscriptions, scheduled statistics collection scripts, and data backup scripts. + +## Step 4: Switch business traffic and rollback + +### 1. Prepare for the switchover + +- Confirm replication status: + + - Monitor the latency of TiCDC Changefeed replication. + - Ensure that the incremental replication throughput is greater than or equal to the peak business write rate. + +- Perform multi-dimensional validation, such as: + + - Ensure that all data validation steps are complete and perform any necessary additional checks. + - Conduct sanity or integration tests on the application in the new cluster. + +### 2. Execute the switchover + +1. Stop application services to prevent the old cluster from handling business traffic. To further restrict access, you can use one of the following methods: + + - Lock user accounts in the old cluster: + + ```sql + ALTER USER ACCOUNT LOCK; + ``` + + - Set the old cluster to read-only mode. It is recommended to restart TiDB nodes in the old cluster to clear active business sessions and prevent connections that have not entered read-only mode: + + ```sql + SET GLOBAL tidb_super_read_only=ON; + ``` + +2. Ensure TiCDC catches up: + + - After setting the old cluster to read-only mode, retrieve the current `up-tso`: + + ```sql + BEGIN; SELECT TIDB_CURRENT_TSO(); ROLLBACK; + ``` + + - Monitor the Changefeed `checkpointTs` to confirm it has surpassed `up-tso`, indicating that TiCDC has completed data replication. + +3. Verify data consistency between the new and old clusters: + + - After TiCDC catches up, obtain the `down-tso` from the new cluster. + - Use the [sync-diff-inspector](/sync-diff-inspector/sync-diff-inspector-overview.md) tool to compare data consistency between the new and old clusters at `up-tso` and `down-tso`. + +4. Pause the forward Changefeed replication task: + + ```shell + tiup ctl:${cluster_version} cdc changefeed pause --server http://${cdc_host}:${cdc_port} -c + ``` + +5. Restart the TiDB nodes in the new cluster to clear the auto-increment ID cache. + +6. Check the operational status of the new cluster using the following methods: + + - Verify that the TiDB version matches the target version: + + ```shell + tiup cluster display + ``` + + - Log into the database and confirm component versions: + + ```sql + SELECT * FROM INFORMATION_SCHEMA.CLUSTER_INFO; + ``` + + - Use Grafana to monitor service status: navigate to [**Overview > Services Port Status**](/grafana-overview-dashboard.md) and confirm that all services are in the **Up** state. + +7. Set up reverse replication from the new cluster to the old cluster. + + 1. Unlock user accounts in the old cluster and restore read-write mode: + + ```sql + ALTER USER ACCOUNT UNLOCK; + SET GLOBAL tidb_super_read_only=OFF; + ``` + + 2. Record the current TSO of the new cluster: + + ```sql + BEGIN; SELECT TIDB_CURRENT_TSO(); ROLLBACK; + ``` + + 3. Configure the reverse replication link and ensure the Changefeed task is running properly: + + - Because business operations are stopped at this stage, you can use the current TSO. + - Ensure that `sink-uri` is set to the address of the old cluster to avoid loopback writing risks. + + ```shell + tiup ctl:${cluster_version} cdc changefeed create --server http://${cdc_host}:${cdc_port} --sink-uri="mysql://${username}:${password}@${tidb_endpoint}:${port}" --config config.toml --start-ts ${tso} + + tiup ctl:${cluster_version} cdc changefeed list --server http://${cdc_host}:${cdc_port} + ``` + +8. Redirect business traffic to the new cluster. + +9. Monitor the load and operational status of the new cluster using the following Grafana panels: + + - [**TiDB Dashboard > Query Summary**](/grafana-tidb-dashboard.md#query-summary): check the Duration, QPS, and Failed Query OPM metrics. + - [**TiDB Dashboard > Server**](/grafana-tidb-dashboard.md#server): monitor the **Connection Count** metric to ensure even distribution of connections across nodes. + +At this point, business traffic has successfully switched to the new cluster, and the TiCDC reverse replication channel is established. + +### 3. Execute emergency rollback + +The rollback plan is as follows: + +- Check data consistency between the new and old clusters regularly to ensure the reverse replication link is operating properly. +- Monitor the system for a specified period, such as one week. If issues occur, switch back to the old cluster. +- After the observation period, remove the reverse replication link and delete the old cluster. + +The following introduces the usage scenario and steps for an emergency rollback, which redirects traffic back to the old cluster: + +- Usage scenario: execute the rollback plan if critical issues cannot be resolved. +- Steps: + + 1. Stop business access to the new cluster. + 2. Reauthorize business accounts and restore read-write access to the old cluster. + 3. Check the reverse replication link, confirm TiCDC has caught up, and verify data consistency between the new and old clusters. + 4. Redirect business traffic back to the old cluster. + +## Step 5: Clean up + +After monitoring the new cluster for a period and confirming stable business operations, you can remove the TiCDC reverse replication and delete the old cluster. + +- Remove the TiCDC reverse replication: + + ```shell + tiup ctl:${cluster_version} cdc changefeed remove --server http://${cdc_host}:${cdc_port} -c + ``` + +- Delete the old cluster. If you choose to retain it, restore `tidb_gc_life_time` to its original value: + + ```sql + -- Restore to the original value before modification. + SET GLOBAL tidb_gc_life_time=10m; + ``` diff --git a/tiflash-deployment-topology.md b/tiflash-deployment-topology.md index 92d12f806af48..9a8c91c19d29f 100644 --- a/tiflash-deployment-topology.md +++ b/tiflash-deployment-topology.md @@ -1,7 +1,6 @@ --- title: TiFlash Deployment Topology summary: Learn the deployment topology of TiFlash based on the minimal TiDB topology. -aliases: ['/docs/dev/tiflash-deployment-topology/'] --- # TiFlash Deployment Topology @@ -20,6 +19,10 @@ TiFlash is a columnar storage engine, and gradually becomes the standard cluster | TiFlash | 1 | 32 VCore 64 GB 2TB (nvme ssd) * 1 | 10.0.1.11 | Default port
    Global directory configuration | | Monitoring & Grafana | 1 | 4 VCore 8GB * 1 500GB (ssd) | 10.0.1.10 | Default port
    Global directory configuration | +> **Note:** +> +> The IP addresses of the instances are given as examples only. In your actual deployment, replace the IP addresses with your actual IP addresses. + ### Topology templates - [The simple template for the TiFlash topology](https://github.com/pingcap/docs/blob/master/config-templates/simple-tiflash.yaml) diff --git a/tiflash-upgrade-guide.md b/tiflash-upgrade-guide.md index cd8ff9b62afc8..90c97d87d39a2 100644 --- a/tiflash-upgrade-guide.md +++ b/tiflash-upgrade-guide.md @@ -1,7 +1,7 @@ --- title: TiFlash Upgrade Guide summary: Learn the precautions when you upgrade TiFlash. -aliases: ['/tidb/dev/tiflash-620-upgrade-guide'] +aliases: ['/tidb/stable/tiflash-620-upgrade-guide'] --- # TiFlash Upgrade Guide @@ -19,15 +19,43 @@ To learn the standard upgrade process, see the following documents: > > - It is not recommended that you upgrade TiDB that includes TiFlash across major versions, for example, from v4.x to v6.x. Instead, you need to upgrade from v4.x to v5.x first, and then to v6.x. > -> - v4.x is near the end of its life cycle. It is recommended that you upgrade to v5.x or later as soon as possible. For more information, see [TiDB Release Support Policy](https://en.pingcap.com/tidb-release-support-policy/). +> - v4.x is near the end of its life cycle. It is recommended that you upgrade to v5.x or later as soon as possible. For more information, see [TiDB Release Support Policy](https://www.pingcap.com/tidb-release-support-policy/). > > - PingCAP does not provide bug fixes for non-LTS versions, such as v6.0. It is recommended that you upgrade to v6.1 and later LTS versions whenever possible. > -> - To upgrade TiFlash from versions earlier than v5.3.0 to v5.3.0 or later, you should stop TiFlash and then upgrade it. The following steps help you upgrade TiFlash without interrupting other components: -> -> - Stop the TiFlash instance: `tiup cluster stop -R tiflash` -> - Upgrade the TiDB cluster without restarting it (only updating the files): `tiup cluster upgrade --offline`, such as `tiup cluster upgrade v5.3.0 --offline` -> - Reload the TiDB cluster: `tiup cluster reload `. After the reload, the TiFlash instance is started and you do not need to manually start it. + +## Upgrade TiFlash using TiUP + +To upgrade TiFlash from versions earlier than v5.3.0 to v5.3.0 or later, you must stop TiFlash and then upgrade it. When you upgrade TiFlash using TiUP, note the following: + +- If the TiUP cluster version is v1.12.0 or later, you cannot stop TiFlash and then upgrade it. If the target version requires a TiUP cluster version of v1.12.0 or later, it is recommended that you first use `tiup cluster:v1.11.3 ` to upgrade TiFlash to an intermediate version, perform an online upgrade of the TiDB cluster, upgrade the TiUP version, and then upgrade the TiDB cluster to the target version directly without stopping it. +- If the TiUP cluster version is earlier than v1.12.0, perform the following steps to upgrade TiFlash. + +The following steps help you use TiUP to upgrade TiFlash without interrupting other components: + +1. Stop the TiFlash instance: + + ```shell + tiup cluster stop -R tiflash + ``` + +2. Upgrade the TiDB cluster without restarting it (only updating the files): + + ```shell + tiup cluster upgrade --offline + ``` + + For example: + + ```shell + tiup cluster upgrade v5.3.0 --offline + ``` + +3. Reload the TiDB cluster. After the reload, the TiFlash instance is started and you do not need to manually start it. + + ```shell + tiup cluster reload + ``` ## From 5.x or v6.0 to v6.1 diff --git a/tiflash/create-tiflash-replicas.md b/tiflash/create-tiflash-replicas.md index 5354df9f029c4..c7dd0dec0edb7 100644 --- a/tiflash/create-tiflash-replicas.md +++ b/tiflash/create-tiflash-replicas.md @@ -19,6 +19,10 @@ The parameter of the above command is described as follows: - `count` indicates the number of replicas. When the value is `0`, the replica is deleted. +> **Note:** +> +> For a [{{{ .starter }}}](https://docs.pingcap.com/tidbcloud/select-cluster-tier#starter) cluster, the `count` of TiFlash replicas can only be `2`. If you set it to `1`, it will be automatically adjusted to `2` for execution. If you set it to a number larger than 2, you will get an error about the replica count. + If you execute multiple DDL statements on the same table, only the last statement is ensured to take effect. In the following example, two DDL statements are executed on the table `tpch50`, but only the second statement (to delete the replica) takes effect. Create two replicas for the table: @@ -130,7 +134,7 @@ Before TiFlash replicas are added, each TiKV instance performs a full table scan ```sql -- The default value for both configurations are 100MiB, i.e. the maximum disk bandwidth used for writing snapshots is no more than 100MiB/s. SET CONFIG tikv `server.snap-io-max-bytes-per-sec` = '300MiB'; - SET CONFIG tiflash `raftstore-proxy.server.snap-max-write-bytes-per-sec` = '300MiB'; + SET CONFIG tiflash `raftstore-proxy.server.snap-io-max-bytes-per-sec` = '300MiB'; ``` After executing these SQL statements, the configuration changes take effect immediately without restarting the cluster. However, since the replication speed is still restricted by the PD limit globally, you cannot observe the acceleration for now. @@ -143,10 +147,10 @@ Before TiFlash replicas are added, each TiKV instance performs a full table scan tiup ctl:v pd -u http://:2379 store limit all engine tiflash 60 add-peer ``` - > In the preceding command, you need to replace `v` with the actual cluster version, such as `v7.4.0` and `:2379` with the address of any PD node. For example: + > In the preceding command, you need to replace `v` with the actual cluster version, such as `v{{{ .tidb-version }}}` and `:2379` with the address of any PD node. For example: > > ```shell - > tiup ctl:v7.4.0 pd -u http://192.168.1.4:2379 store limit all engine tiflash 60 add-peer + > tiup ctl:v{{{ .tidb-version }}} pd -u http://192.168.1.4:2379 store limit all engine tiflash 60 add-peer > ``` Within a few minutes, you will observe a significant increase in CPU and disk IO resource usage of the TiFlash nodes, and TiFlash should create replicas faster. At the same time, the TiKV nodes' CPU and disk IO resource usage increases as well. @@ -169,7 +173,7 @@ Before TiFlash replicas are added, each TiKV instance performs a full table scan ```sql SET CONFIG tikv `server.snap-io-max-bytes-per-sec` = '100MiB'; - SET CONFIG tiflash `raftstore-proxy.server.snap-max-write-bytes-per-sec` = '100MiB'; + SET CONFIG tiflash `raftstore-proxy.server.snap-io-max-bytes-per-sec` = '100MiB'; ``` ## Set available zones @@ -209,47 +213,50 @@ When configuring replicas, if you need to distribute TiFlash replicas to multipl Note that the `flash.proxy.labels` configuration in earlier versions cannot handle special characters in the available zone name correctly. It is recommended to use the `server.labels` in `learner_config` to configure the name of an available zone. -2. After starting a cluster, specify the labels when creating replicas. +2. After starting a cluster, specify the number of TiFlash replicas for high availability. The syntax is as follows: ```sql - ALTER TABLE table_name SET TIFLASH REPLICA count LOCATION LABELS location_labels; + ALTER TABLE table_name SET TIFLASH REPLICA count; ``` For example: ```sql - ALTER TABLE t SET TIFLASH REPLICA 2 LOCATION LABELS "zone"; + ALTER TABLE t SET TIFLASH REPLICA 2; ``` -3. PD schedules the replicas based on the labels. In this example, PD respectively schedules two replicas of the table `t` to two available zones. You can use pd-ctl to view the scheduling. +3. PD schedules the replicas of the table `t` to different availability zones based on the `server.labels` in the TiFlash node's `learner_config` and the number (`count`) of the table's replicas, ensuring availability. For more information, see [Schedule Replicas by Topology Labels](https://docs.pingcap.com/tidb/stable/schedule-replicas-by-topology-labels/). You can use the following SQL statement to verify the distribution of a table's Regions across TiFlash nodes: - ```shell - > tiup ctl:v pd -u http://:2379 store - - ... - "address": "172.16.5.82:23913", - "labels": [ - { "key": "engine", "value": "tiflash"}, - { "key": "zone", "value": "z1" } - ], - "region_count": 4, - - ... - "address": "172.16.5.81:23913", - "labels": [ - { "key": "engine", "value": "tiflash"}, - { "key": "zone", "value": "z1" } - ], - "region_count": 5, - ... - - "address": "172.16.5.85:23913", - "labels": [ - { "key": "engine", "value": "tiflash"}, - { "key": "zone", "value": "z2" } - ], - "region_count": 9, - ... + ```sql + -- Non-partitioned table + SELECT table_id, p.store_id, address, COUNT(p.region_id) + FROM + information_schema.tikv_region_status r, + information_schema.tikv_region_peers p, + information_schema.tikv_store_status s + WHERE + r.db_name = 'test' + AND r.table_name = 'table_to_check' + AND r.region_id = p.region_id + AND p.store_id = s.store_id + AND JSON_EXTRACT(s.label, '$[0].value') = 'tiflash' + GROUP BY table_id, p.store_id, address; + + -- Partitioned table + SELECT table_id, r.partition_name, p.store_id, address, COUNT(p.region_id) + FROM + information_schema.tikv_region_status r, + information_schema.tikv_region_peers p, + information_schema.tikv_store_status s + WHERE + r.db_name = 'test' + AND r.table_name = 'table_to_check' + AND r.partition_name LIKE 'p202312%' + AND r.region_id = p.region_id + AND p.store_id = s.store_id + AND JSON_EXTRACT(s.label, '$[0].value') = 'tiflash' + GROUP BY table_id, r.partition_name, p.store_id, address + ORDER BY table_id, r.partition_name, p.store_id; ``` @@ -259,3 +266,7 @@ For more information about scheduling replicas by using labels, see [Schedule Re TiFlash supports configuring the replica selection strategy for different zones. For more information, see [`tiflash_replica_read`](/system-variables.md#tiflash_replica_read-new-in-v730). + +> **Note:** +> +> In the syntax `ALTER TABLE table_name SET TIFLASH REPLICA count LOCATION LABELS location_labels;`, if you specify multiple labels for `location_labels`, TiDB cannot parse them correctly to set placement rules. Therefore, do not use `LOCATION LABELS` to configure TiFlash replicas. diff --git a/tiflash/maintain-tiflash.md b/tiflash/maintain-tiflash.md index 3831ac2bc99c1..ab9d32dd0d612 100644 --- a/tiflash/maintain-tiflash.md +++ b/tiflash/maintain-tiflash.md @@ -1,7 +1,6 @@ --- title: Maintain a TiFlash Cluster summary: Learn common operations when you maintain a TiFlash cluster. -aliases: ['/docs/dev/tiflash/maintain-tiflash/','/docs/dev/reference/tiflash/maintain/'] --- # Maintain a TiFlash Cluster diff --git a/tiflash/monitor-tiflash.md b/tiflash/monitor-tiflash.md index f1cdb07cf1eb1..710fe2b66cb1f 100644 --- a/tiflash/monitor-tiflash.md +++ b/tiflash/monitor-tiflash.md @@ -1,7 +1,6 @@ --- title: Monitor the TiFlash Cluster summary: Learn the monitoring items of TiFlash. -aliases: ['/docs/dev/tiflash/monitor-tiflash/','/docs/dev/reference/tiflash/monitor/'] --- # Monitor the TiFlash Cluster diff --git a/tiflash/tiflash-alert-rules.md b/tiflash/tiflash-alert-rules.md index a28f9ea713ead..993a78c602c01 100644 --- a/tiflash/tiflash-alert-rules.md +++ b/tiflash/tiflash-alert-rules.md @@ -1,7 +1,6 @@ --- title: TiFlash Alert Rules summary: Learn the alert rules of the TiFlash cluster. -aliases: ['/docs/dev/tiflash/tiflash-alert-rules/','/docs/dev/reference/tiflash/alert-rules/'] --- # TiFlash Alert Rules diff --git a/tiflash/tiflash-command-line-flags.md b/tiflash/tiflash-command-line-flags.md index c76e0b4f2735d..8cfbb93e757b0 100644 --- a/tiflash/tiflash-command-line-flags.md +++ b/tiflash/tiflash-command-line-flags.md @@ -1,7 +1,6 @@ --- title: TiFlash Command-line Flags summary: Learn the command-line startup flags of TiFlash. -aliases: ['/docs/dev/tiflash/tiflash-command-line-flags/'] --- # TiFlash Command-Line Flags diff --git a/tiflash/tiflash-configuration.md b/tiflash/tiflash-configuration.md index e7c63922e7107..009dc5c8fd500 100644 --- a/tiflash/tiflash-configuration.md +++ b/tiflash/tiflash-configuration.md @@ -1,7 +1,6 @@ --- title: Configure TiFlash summary: Learn how to configure TiFlash. -aliases: ['/docs/dev/tiflash/tiflash-configuration/','/docs/dev/reference/tiflash/configuration/'] --- # Configure TiFlash @@ -129,7 +128,7 @@ delta_index_cache_size = 0 ## auto_tune_sec indicates the interval of automatic tuning. The unit is seconds. If the value of auto_tune_sec is 0, the automatic tuning is disabled. # auto_tune_sec = 5 - ## The following configuration items only take effect for the TiFlash disaggregated storage and compute architecture mode. For details, see documentation at https://docs.pingcap.com/tidb/dev/tiflash-disaggregated-and-s3. + ## The following configuration items only take effect for the TiFlash disaggregated storage and compute architecture mode. For details, see documentation at https://docs.pingcap.com/tidb/v7.5/tiflash-disaggregated-and-s3. # [storage.s3] # endpoint: http://s3.{region}.amazonaws.com # S3 endpoint address # bucket: mybucket # TiFlash stores all data in this bucket @@ -141,8 +140,8 @@ delta_index_cache_size = 0 # capacity: 858993459200 # 800 GiB [flash] - tidb_status_addr = TiDB status port and address. # Multiple addresses are separated with commas. - service_addr = The listening address of TiFlash Raft services and coprocessor services. + ## The listening address of TiFlash coprocessor services. + service_addr = "0.0.0.0:3930" ## Introduced in v7.4.0. When the gap between the `applied_index` advanced by the current Raft state machine and the `applied_index` at the last disk spilling exceeds `compact_log_min_gap`, TiFlash executes the `CompactLog` command from TiKV and spills data to disk. Increasing this gap might reduce the disk spilling frequency of TiFlash, thus reducing read latency in random write scenarios, but it might also increase memory overhead. Decreasing this gap might increase the disk spilling frequency of TiFlash, thus alleviating memory pressure in TiFlash. However, at this stage, the disk spilling frequency of TiFlash will not be higher than that of TiKV, even if this gap is set to 0. ## It is recommended to keep the default value. @@ -152,33 +151,34 @@ delta_index_cache_size = 0 # compact_log_min_rows = 40960 # 40k # compact_log_min_bytes = 33554432 # 32MB - ## The following configuration item only takes effect for the TiFlash disaggregated storage and compute architecture mode. For details, see documentation at https://docs.pingcap.com/tidb/dev/tiflash-disaggregated-and-s3. + ## The following configuration item only takes effect for the TiFlash disaggregated storage and compute architecture mode. For details, see documentation at https://docs.pingcap.com/tidb/v7.5/tiflash-disaggregated-and-s3. # disaggregated_mode = tiflash_write # The supported mode is `tiflash_write` or `tiflash_compute. -## Multiple TiFlash nodes elect a master to add or delete placement rules to PD, -## and the configurations in flash.flash_cluster control this process. -[flash.flash_cluster] - refresh_interval = Master regularly refreshes the valid period. - update_rule_interval = Master regularly gets the status of TiFlash replicas and interacts with PD. - master_ttl = The valid period of the elected master. - cluster_manager_path = The absolute path of the pd buddy directory. - log = The pd buddy log path. - [flash.proxy] - addr = The listening address of proxy. If it is left empty, 127.0.0.1:20170 is used by default. - advertise-addr = The external access address of addr. If it is left empty, "addr" is used by default. - data-dir = The data storage path of proxy. - config = The configuration file path of proxy. - log-file = The log path of proxy. - log-level = The log level of proxy. "info" is used by default. - status-addr = The listening address from which the proxy pulls metrics | status information. If it is left empty, 127.0.0.1:20292 is used by default. - advertise-status-addr = The external access address of status-addr. If it is left empty, "status-addr" is used by default. + ## The listening address of proxy. If it is left empty, 127.0.0.1:20170 is used by default. + addr = "127.0.0.1:20170" + ## The external access address of addr. If it is left empty, "addr" is used by default. + ## Should guarantee that other nodes can access through `advertise-addr` when you deploy the cluster on multiple nodes. + advertise-addr = "" + ## The listening address from which the proxy pulls metrics or status information. If it is left empty, 127.0.0.1:20292 is used by default. + status-addr = "127.0.0.1:20292" + ## The external access address of status-addr. If it is left empty, the value of "status-addr" is used by default. + ## Should guarantee that other nodes can access through `advertise-status-addr` when you deploy the cluster on multiple nodes. + advertise-status-addr = "" + ## The data storage path of proxy. + data-dir = "/tidb-data/tiflash-9000/flash" + ## The configuration file path of proxy. + config = "/tidb-deploy/tiflash-9000/conf/tiflash-learner.toml" + ## The log path of proxy. + log-file = "/tidb-deploy/tiflash-9000/log/tiflash_tikv.log" + ## The log level of proxy (available options: "trace", "debug", "info", "warn", "error"). The default value is "info" + # log-level = "info" [logger] - ## log level (available options: "trace", "debug", "info", "warn", "error"). The default value is "debug". - level = "debug" - log = TiFlash log path - errorlog = TiFlash error log path + ## log level (available options: "trace", "debug", "info", "warn", "error"). Starting from v7.5.1, the default value changes from "debug" to "info". + level = "info" + log = "/tidb-deploy/tiflash-9000/log/tiflash.log" + errorlog = "/tidb-deploy/tiflash-9000/log/tiflash_error.log" ## Size of a single log file. The default value is "100M". size = "100M" ## Maximum number of log files to save. The default value is 10. @@ -220,6 +220,13 @@ delta_index_cache_size = 0 ## New in v5.0. This item specifies the maximum number of cop requests that TiFlash Coprocessor executes at the same time. If the number of requests exceeds the specified value, the exceeded requests will queue. If the configuration value is set to 0 or not set, the default value is used, which is twice the number of physical cores. cop_pool_size = 0 + + ## New in v5.0. This item specifies the maximum number of cop requests that TiFlash Coprocessor handles at the same time, including the requests being executed and the requests waiting in the queue. If the number of requests exceeds the specified value, the error "TiFlash Server is Busy" is returned. -1 indicates no limit; 0 indicates using the default value, which is 10 * cop_pool_size. + cop_pool_handle_limit = 0 + + ## New in v5.0. This item specifies the maximum time that a cop request can queue in TiFlash. If a cop request waits in the queue for a time longer than the value specified by this configuration, the error "TiFlash Server is Busy" is returned. A value less than or equal to 0 indicates no limit. + cop_pool_max_queued_seconds = 15 + ## New in v5.0. This item specifies the maximum number of batch requests that TiFlash Coprocessor executes at the same time. If the number of requests exceeds the specified value, the exceeded requests will queue. If the configuration value is set to 0 or not set, the default value is used, which is twice the number of physical cores. batch_cop_pool_size = 0 ## New in v6.1.0. This item specifies the number of requests that TiFlash can concurrently process when it receives ALTER TABLE ... COMPACT from TiDB. @@ -250,6 +257,15 @@ delta_index_cache_size = 0 ## New in v7.4.0. This item controls whether to enable the TiFlash resource control feature. When it is set to true, TiFlash uses the pipeline execution model. enable_resource_control = true + ## New in v6.0.0. This item is used for the MinTSO scheduler. It specifies the maximum number of threads that one resource group can use. The default value is 5000. For details about the MinTSO scheduler, see https://docs.pingcap.com/tidb/v7.5/tiflash-mintso-scheduler. + task_scheduler_thread_soft_limit = 5000 + + ## New in v6.0.0. This item is used for the MinTSO scheduler. It specifies the maximum number of threads in the global scope. The default value is 10000. For details about the MinTSO scheduler, see https://docs.pingcap.com/tidb/v7.5/tiflash-mintso-scheduler. + task_scheduler_thread_hard_limit = 10000 + + ## New in v6.4.0. This item is used for the MinTSO scheduler. It specifies the maximum number of queries that can run simultaneously in a TiFlash instance. The default value is 0, which means twice the number of vCPUs. For details about the MinTSO scheduler, see https://docs.pingcap.com/tidb/v7.5/tiflash-mintso-scheduler. + task_scheduler_active_set_soft_limit = 0 + ## Security settings take effect starting from v4.0.5. [security] ## New in v5.0. This configuration item enables or disables log redaction. If the configuration value @@ -283,11 +299,6 @@ delta_index_cache_size = 0 ## If you set it to 0, the multi-thread optimization is disabled. snap-handle-pool-size = 2 - ## The shortest interval at which Raft store persists WAL. - ## You can properly increase the latency to reduce IOPS usage. - ## The default value is "4ms". - ## If you set it to 0ms, the optimization is disabled. - store-batch-retry-recv-timeout = "4ms" [security] ## New in v5.0. This configuration item enables or disables log redaction. ## If the configuration value is set to true, @@ -303,10 +314,10 @@ delta_index_cache_size = 0 data-key-rotation-period = "168h" # 7 days [security.encryption.master-key] - ## Specifies the master key if encryption is enabled. To learn how to configure a master key, see Configure encryption: https://docs.pingcap.com/tidb/dev/encryption-at-rest#configure-encryption . + ## Specifies the master key if encryption is enabled. To learn how to configure a master key, see Configure encryption: https://docs.pingcap.com/tidb/v7.5/encryption-at-rest#configure-encryption . [security.encryption.previous-master-key] - ## Specifies the old master key when rotating the new master key. The configuration format is the same as that of `master-key`. To learn how to configure a master key, see Configure encryption: https://docs.pingcap.com/tidb/dev/encryption-at-rest#configure-encryption . + ## Specifies the old master key when rotating the new master key. The configuration format is the same as that of `master-key`. To learn how to configure a master key, see Configure encryption: https://docs.pingcap.com/tidb/v7.5/encryption-at-rest#configure-encryption . ``` In addition to the items above, other parameters are the same as those of TiKV. Note that the `label` whose key is `engine` is reserved and cannot be configured manually. diff --git a/tiflash/tiflash-disaggregated-and-s3.md b/tiflash/tiflash-disaggregated-and-s3.md index 11b9bd17e3f7f..856d64dbe2a6e 100644 --- a/tiflash/tiflash-disaggregated-and-s3.md +++ b/tiflash/tiflash-disaggregated-and-s3.md @@ -64,9 +64,9 @@ TiFlash disaggregated storage and compute architecture is suitable for cost-effe ``` ```shell - tiup cluster scale-in mycuster -R tiflash # Remove all TiFlash nodes - tiup cluster display mycluster # Wait for all TiFlash nodes to enter the Tombstone state - tiup cluster prune mycluster # Remove all TiFlash nodes in the Tombstone state + tiup cluster scale-in mycluster -N 'node0,node1...' # Remove all TiFlash nodes + tiup cluster display mycluster # Wait for all TiFlash nodes to enter the Tombstone state + tiup cluster prune mycluster # Remove all TiFlash nodes in the Tombstone state ``` ## Usage diff --git a/tiflash/tiflash-late-materialization.md b/tiflash/tiflash-late-materialization.md index 6493753df0a22..3ad8ea60a87c0 100644 --- a/tiflash/tiflash-late-materialization.md +++ b/tiflash/tiflash-late-materialization.md @@ -14,7 +14,7 @@ TiFlash late materialization is an optimization method to accelerate queries in - When it is disabled, to process a `SELECT` statement with filter conditions (`WHERE` clause), TiFlash reads all the data from the columns required by the query, and then filters and aggregates the data based on the query conditions. - When it is enabled, TiFlash supports pushing down part of the filter conditions to the TableScan operator. That is, TiFlash first scans the column data related to the filter conditions that are pushed down to the TableScan operator, filters the rows that meet the condition, and then scans the other column data of these rows for further calculation, thereby reducing IO scans and computations of data processing. -To improve the performance of certain queries in OLAP scenarios, starting from v7.1.0, the TiFlash late materialization feature is enabled by default. The TiDB optimizer can determine which filter conditions to be pushed down based on statistics and filter conditions, and prioritize pushing down the filter conditions with high filtration rates. For detailed algorithms, see the [RFC document](https://github.com/pingcap/tidb/tree/master/docs/design/2022-12-06-support-late-materialization.md). +To improve the performance of certain queries in OLAP scenarios, starting from v7.1.0, the TiFlash late materialization feature is enabled by default. The TiDB optimizer can determine which filter conditions to be pushed down based on statistics and filter conditions, and prioritize pushing down the filter conditions with high filtration rates. For detailed algorithms, see the [RFC document](https://github.com/pingcap/tidb/tree/release-7.5/docs/design/2022-12-06-support-late-materialization.md). For example: diff --git a/tiflash/tiflash-mintso-scheduler.md b/tiflash/tiflash-mintso-scheduler.md new file mode 100644 index 0000000000000..0fc21995580cd --- /dev/null +++ b/tiflash/tiflash-mintso-scheduler.md @@ -0,0 +1,68 @@ +--- +title: TiFlash MinTSO Scheduler +summary: Learn the implementation principles of the TiFlash MinTSO Scheduler. +--- + +# TiFlash MinTSO Scheduler + +The TiFlash MinTSO scheduler is a distributed scheduler for [MPP](/glossary.md#mpp) tasks in TiFlash. This document describes the implementation principles of the TiFlash MinTSO scheduler. + +## Background + +When processing an MPP query, TiDB splits the query into one or more MPP tasks and sends these MPP tasks to the corresponding TiFlash nodes for compilation and execution. Before TiFlash uses the [pipeline execution model](/tiflash/tiflash-pipeline-model.md), TiFlash needs to use several threads to execute each MPP task, with the specific number of threads depending on the complexity of the MPP task and the concurrency parameters set in TiFlash. + +In high concurrency scenarios, TiFlash nodes receive multiple MPP tasks simultaneously. If the execution of MPP tasks is not controlled, the number of threads that TiFlash needs to request from the system will increase linearly along with the increasing number of MPP tasks. Too many threads can affect the execution efficiency of TiFlash, and because the operating system itself supports a limited number of threads, TiFlash will encounter errors when it requests more threads than the operating system can provide. + +To improve TiFlash's processing capability in high concurrency scenarios, an MPP task scheduler needs to be introduced into TiFlash. + +## Implementation principles + +As mentioned in the [background](#background), the initial purpose of introducing the TiFlash task scheduler is to control the number of threads used during MPP query execution. A simple scheduling strategy is to specify the maximum number of threads TiFlash can request. For each MPP task, the scheduler decides whether the MPP task can be scheduled based on the current number of threads used by the system and the expected number of threads the MPP task will use: + +![TiFlash MinTSO Scheduler v1](/media/tiflash/tiflash_mintso_v1.png) + +Although the preceding scheduling strategy can effectively control the number of system threads, an MPP task is not the smallest independent execution unit, and dependencies exist between different MPP tasks: + +```sql +EXPLAIN SELECT count(*) FROM t0 a JOIN t0 b ON a.id = b.id; +``` + +``` ++--------------------------------------------+----------+--------------+---------------+----------------------------------------------------------+ +| id | estRows | task | access object | operator info | ++--------------------------------------------+----------+--------------+---------------+----------------------------------------------------------+ +| HashAgg_44 | 1.00 | root | | funcs:count(Column#8)->Column#7 | +| └─TableReader_46 | 1.00 | root | | MppVersion: 2, data:ExchangeSender_45 | +| └─ExchangeSender_45 | 1.00 | mpp[tiflash] | | ExchangeType: PassThrough | +| └─HashAgg_13 | 1.00 | mpp[tiflash] | | funcs:count(1)->Column#8 | +| └─Projection_43 | 12487.50 | mpp[tiflash] | | test.t0.id | +| └─HashJoin_42 | 12487.50 | mpp[tiflash] | | inner join, equal:[eq(test.t0.id, test.t0.id)] | +| ├─ExchangeReceiver_22(Build) | 9990.00 | mpp[tiflash] | | | +| │ └─ExchangeSender_21 | 9990.00 | mpp[tiflash] | | ExchangeType: Broadcast, Compression: FAST | +| │ └─Selection_20 | 9990.00 | mpp[tiflash] | | not(isnull(test.t0.id)) | +| │ └─TableFullScan_19 | 10000.00 | mpp[tiflash] | table:a | pushed down filter:empty, keep order:false, stats:pseudo | +| └─Selection_24(Probe) | 9990.00 | mpp[tiflash] | | not(isnull(test.t0.id)) | +| └─TableFullScan_23 | 10000.00 | mpp[tiflash] | table:b | pushed down filter:empty, keep order:false, stats:pseudo | ++--------------------------------------------+----------+--------------+---------------+----------------------------------------------------------+ +``` + +For example, the preceding query generates two MPP tasks on each TiFlash node, where the MPP task containing the `ExchangeSender_45` executor depends on the MPP task containing the `ExchangeSender_21` executor. In high concurrency scenarios, if the scheduler schedules the MPP task containing `ExchangeSender_45` for each query, the system will enter a deadlock state. + +To avoid deadlock, TiFlash introduces the following two levels of thread limits: + +* thread_soft_limit: used to limit the number of threads used by the system. For specific MPP tasks, this limit can be broken to avoid deadlock. +* thread_hard_limit: used to protect the system. Once the number of threads used by the system exceeds the hard limit, TiFlash will report an error to avoid deadlock. + +The soft limit and hard limit work together to avoid deadlock as follows: the soft limit restricts the total number of threads used by all queries, enabling full use of resources while avoiding thread resource exhaustion; the hard limit ensures that in any situation, at least one query in the system can break the soft limit and continue to acquire thread resources and run, thus avoiding deadlock. As long as the number of threads does not exceed the hard limit, there will always be one query in the system where all its MPP tasks can be executed normally, thus preventing deadlock. + +The goal of the MinTSO scheduler is to control the number of system threads while ensuring that there is always one and only one special query in the system, where all its MPP tasks can be scheduled. The MinTSO scheduler is a fully distributed scheduler, with each TiFlash node scheduling MPP tasks based only on its own information. Therefore, all MinTSO schedulers on TiFlash nodes need to identify the same "special" query. In TiDB, each query carries a read timestamp (`start_ts`), and the MinTSO scheduler defines the "special" query as the query with the smallest `start_ts` on the current TiFlash node. Based on the principle that the global minimum is also the local minimum, the "special" query selected by all TiFlash nodes must be the same, called the MinTSO query. + +The scheduling process of the MinTSO Scheduler is as follows: + +![TiFlash MinTSO Scheduler v2](/media/tiflash/tiflash_mintso_v2.png) + +By introducing soft limit and hard limit, the MinTSO scheduler effectively avoids system deadlock while controlling the number of system threads. In high concurrency scenarios, however, most queries might only have part of their MPP tasks scheduled. Queries with only part of MPP tasks scheduled cannot execute normally, leading to low system execution efficiency. To avoid this situation, TiFlash introduces a query-level limit for the MinTSO scheduler, called active_set_soft_limit. This limit allows only MPP tasks of up to active_set_soft_limit queries to participate in scheduling; MPP tasks of other queries do not participate in scheduling, and only after the current queries finish can new queries participate in scheduling. This limit is only a soft limit because for the MinTSO query, all its MPP tasks can be scheduled directly as long as the number of system threads does not exceed the hard limit. + +## See also + +- [Configure TiFlash](/tiflash/tiflash-configuration.md): learn how to configure the MinTSO scheduler. diff --git a/tiflash/tiflash-overview.md b/tiflash/tiflash-overview.md index 2999f51cae4f4..adf6848f54371 100644 --- a/tiflash/tiflash-overview.md +++ b/tiflash/tiflash-overview.md @@ -1,7 +1,6 @@ --- title: TiFlash Overview summary: Learn the architecture and key features of TiFlash. -aliases: ['/docs/dev/tiflash/tiflash-overview/','/docs/dev/reference/tiflash/overview/','/docs/dev/tiflash/use-tiflash/','/docs/dev/reference/tiflash/use-tiflash/','/tidb/dev/use-tiflash'] --- # TiFlash Overview @@ -22,7 +21,7 @@ With TiDB Cloud, you can create an HTAP cluster easily by specifying one or more The above figure is the architecture of TiDB in its HTAP form, including TiFlash nodes. -TiFlash provides the columnar storage, with a layer of coprocessors efficiently implemented by ClickHouse. Similar to TiKV, TiFlash also has a Multi-Raft system, which supports replicating and distributing data in the unit of Region (see [Data Storage](https://en.pingcap.com/blog/tidb-internal-data-storage/) for details). +TiFlash provides the columnar storage, with a layer of coprocessors efficiently implemented by ClickHouse. Similar to TiKV, TiFlash also has a Multi-Raft system, which supports replicating and distributing data in the unit of Region (see [Data Storage](https://www.pingcap.com/blog/tidb-internal-data-storage/) for details). TiFlash conducts real-time replication of data in the TiKV nodes at a low cost that does not block writes in TiKV. Meanwhile, it provides the same read consistency as in TiKV and ensures that the latest data is read. The Region replica in TiFlash is logically identical to those in TiKV, and is split and merged along with the Leader replica in TiKV at the same time. diff --git a/tiflash/tiflash-results-materialization.md b/tiflash/tiflash-results-materialization.md index 82f02699ad02b..afec777dde1b9 100644 --- a/tiflash/tiflash-results-materialization.md +++ b/tiflash/tiflash-results-materialization.md @@ -93,8 +93,8 @@ CREATE TABLE daily_data ( customer_id VARCHAR(20), -- Customer ID daily_fee DECIMAL(20,2)); -- Amount of fee for per day -ALTER TABLE detail_data SET TIFLASH REPLICA 1; -ALTER TABLE daily_data SET TIFLASH REPLICA 1; +ALTER TABLE detail_data SET TIFLASH REPLICA 2; +ALTER TABLE daily_data SET TIFLASH REPLICA 2; -- ... (detail_data table continues updating) INSERT INTO detail_data(ts,customer_id,detail_fee) VALUES @@ -103,16 +103,21 @@ INSERT INTO detail_data(ts,customer_id,detail_fee) VALUES ('2023-1-3 12:2:3', 'cus002', 2200.86), ('2023-1-4 12:2:3', 'cus003', 2020.86), ('2023-1-5 12:2:3', 'cus003', 1200.86), -('2023-1-6 12:2:3', 'cus002', 20.86); +('2023-1-6 12:2:3', 'cus002', 20.86), +('2023-1-7 12:2:3', 'cus004', 120.56), +('2023-1-8 12:2:3', 'cus005', 320.16); + +-- Execute the following SQL statement 13 times to insert a cumulative total of 65,536 rows into the table. +INSERT INTO detail_data SELECT * FROM detail_data; ``` Save daily analysis results: ```sql -SET @@tidb_enable_tiflash_read_for_write_stmt=ON; +SET @@sql_mode='NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO'; INSERT INTO daily_data (rec_date, customer_id, daily_fee) -SELECT DATE(ts), customer_id, sum(detail_fee) FROM detail_data WHERE DATE(ts) = CURRENT_DATE() GROUP BY DATE(ts), customer_id; +SELECT DATE(ts), customer_id, sum(detail_fee) FROM detail_data WHERE DATE(ts) > DATE('2023-1-1 12:2:3') GROUP BY DATE(ts), customer_id; ``` Analyze monthly data based on daily analysis data: diff --git a/tiflash/tiflash-spill-disk.md b/tiflash/tiflash-spill-disk.md index d1809951aac59..00d22eaf80006 100644 --- a/tiflash/tiflash-spill-disk.md +++ b/tiflash/tiflash-spill-disk.md @@ -17,8 +17,8 @@ Starting from v7.0.0, TiFlash supports spilling intermediate data to disk to rel TiFlash provides two triggering mechanisms for spilling data to disk. -* Operator-level spilling: by specifing the data spilling threshold for each operator, you can control when TiFlash spills data of that operator to disk. -* Query-level spilling: by specifing the maximum memory usage of a query on a TiFlash node and the memory ratio for spilling, you can control when TiFlash spills data of supported operators in a query to disk as needed. +* Operator-level spilling: by specifying the data spilling threshold for each operator, you can control when TiFlash spills data of that operator to disk. +* Query-level spilling: by specifying the maximum memory usage of a query on a TiFlash node and the memory ratio for spilling, you can control when TiFlash spills data of supported operators in a query to disk as needed. ### Operator-level spilling diff --git a/tiflash/tiflash-supported-pushdown-calculations.md b/tiflash/tiflash-supported-pushdown-calculations.md index 6740d6c5a95f1..499b89ac607c3 100644 --- a/tiflash/tiflash-supported-pushdown-calculations.md +++ b/tiflash/tiflash-supported-pushdown-calculations.md @@ -39,7 +39,7 @@ TiFlash supports the following push-down expressions: | [Logical functions](/functions-and-operators/control-flow-functions.md) and [operators](/functions-and-operators/operators.md) | `AND`, `OR`, `NOT`, `CASE WHEN`, `IF()`, `IFNULL()`, `ISNULL()`, `IN`, `LIKE`, `ILIKE`, `COALESCE`, `IS` | | [Bitwise operations](/functions-and-operators/bit-functions-and-operators.md) | `&` (bitand), \| (bitor), `~` (bitneg), `^` (bitxor) | | [String functions](/functions-and-operators/string-functions.md) | `SUBSTR()`, `CHAR_LENGTH()`, `REPLACE()`, `CONCAT()`, `CONCAT_WS()`, `LEFT()`, `RIGHT()`, `ASCII()`, `LENGTH()`, `TRIM()`, `LTRIM()`, `RTRIM()`, `POSITION()`, `FORMAT()`, `LOWER()`, `UCASE()`, `UPPER()`, `SUBSTRING_INDEX()`, `LPAD()`, `RPAD()`, `STRCMP()` | -| [Regular expression functions and operators](/functions-and-operators/string-functions.md) | `REGEXP`, `REGEXP_LIKE()`, `REGEXP_INSTR()`, `REGEXP_SUBSTR()`, `REGEXP_REPLACE()` | +| [Regular expression functions and operators](/functions-and-operators/string-functions.md) | `REGEXP`, `REGEXP_LIKE()`, `REGEXP_INSTR()`, `REGEXP_SUBSTR()`, `REGEXP_REPLACE()`, `RLIKE` | | [Date functions](/functions-and-operators/date-and-time-functions.md) | `DATE_FORMAT()`, `TIMESTAMPDIFF()`, `FROM_UNIXTIME()`, `UNIX_TIMESTAMP(int)`, `UNIX_TIMESTAMP(decimal)`, `STR_TO_DATE(date)`, `STR_TO_DATE(datetime)`, `DATEDIFF()`, `YEAR()`, `MONTH()`, `DAY()`, `EXTRACT(datetime)`, `DATE()`, `HOUR()`, `MICROSECOND()`, `MINUTE()`, `SECOND()`, `SYSDATE()`, `DATE_ADD/ADDDATE(datetime, int)`, `DATE_ADD/ADDDATE(string, int/real)`, `DATE_SUB/SUBDATE(datetime, int)`, `DATE_SUB/SUBDATE(string, int/real)`, `QUARTER()`, `DAYNAME()`, `DAYOFMONTH()`, `DAYOFWEEK()`, `DAYOFYEAR()`, `LAST_DAY()`, `MONTHNAME()`, `TO_SECONDS()`, `TO_DAYS()`, `FROM_DAYS()`, `WEEKOFYEAR()` | [JSON function](/functions-and-operators/json-functions.md) | `JSON_LENGTH()`, `->`, `->>`, `JSON_EXTRACT()` | | [Conversion functions](/functions-and-operators/cast-functions-and-operators.md) | `CAST(int AS DOUBLE), CAST(int AS DECIMAL)`, `CAST(int AS STRING)`, `CAST(int AS TIME)`, `CAST(double AS INT)`, `CAST(double AS DECIMAL)`, `CAST(double AS STRING)`, `CAST(double AS TIME)`, `CAST(string AS INT)`, `CAST(string AS DOUBLE), CAST(string AS DECIMAL)`, `CAST(string AS TIME)`, `CAST(decimal AS INT)`, `CAST(decimal AS STRING)`, `CAST(decimal AS TIME)`, `CAST(time AS INT)`, `CAST(time AS DECIMAL)`, `CAST(time AS STRING)`, `CAST(time AS REAL)` | diff --git a/tiflash/troubleshoot-tiflash.md b/tiflash/troubleshoot-tiflash.md index d2c0b6c139e8c..7bb2ad80b73ff 100644 --- a/tiflash/troubleshoot-tiflash.md +++ b/tiflash/troubleshoot-tiflash.md @@ -1,7 +1,6 @@ --- title: Troubleshoot a TiFlash Cluster summary: Learn common operations when you troubleshoot a TiFlash cluster. -aliases: ['/docs/dev/tiflash/troubleshoot-tiflash/'] --- # Troubleshoot a TiFlash Cluster @@ -75,7 +74,8 @@ This is because TiFlash is in an abnormal state caused by configuration errors o > **Note:** > - > After the [placement rules](/configure-placement-rules.md) feature is enabled, the previously configured `max-replicas` and `location-labels` no longer take effect. To adjust the replica policy, use the interface related to placement rules. + > - When [Placement Rules](/configure-placement-rules.md) are enabled and multiple rules exist, the previously configured [`max-replicas`](/pd-configuration-file.md#max-replicas), [`location-labels`](/pd-configuration-file.md#location-labels), and [`isolation-level`](/pd-configuration-file.md#isolation-level) no longer take effect. To adjust the replica policy, use the interface related to Placement Rules. + > - When [Placement Rules](/configure-placement-rules.md) are enabled and only one default rule exists, TiDB will automatically update this default rule when `max-replicas`, `location-labels`, or `isolation-level` configurations are changed. 6. Check whether the remaining disk space of the machine (where `store` of the TiFlash node is) is sufficient. By default, when the remaining disk space is less than 20% of the `store` capacity (which is controlled by the `low-space-ratio` parameter), PD cannot schedule data to this TiFlash node. diff --git a/tiflash/tune-tiflash-performance.md b/tiflash/tune-tiflash-performance.md index f17c9b6176cfd..30e62c924a755 100644 --- a/tiflash/tune-tiflash-performance.md +++ b/tiflash/tune-tiflash-performance.md @@ -1,7 +1,6 @@ --- title: Tune TiFlash Performance summary: Learn how to tune the performance of TiFlash by planning machine resources and tuning TiDB parameters. -aliases: ['/docs/dev/tiflash/tune-tiflash-performance/','/docs/dev/reference/tiflash/tune-performance/'] --- # Tune TiFlash Performance @@ -230,7 +229,7 @@ ALTER TABLE employees COMPACT PARTITION pNorth, pEast TIFLASH REPLICA; ### Replace Shuffled Hash Join with Broadcast Hash Join -For `Join` operations with small tables, the Broadcast Hash Join algorithm can avoid transfering large tables, thereby improving the computing performance. +For `Join` operations with small tables, the Broadcast Hash Join algorithm can avoid transferring large tables, thereby improving the computing performance. - The [`tidb_broadcast_join_threshold_size`](/system-variables.md#tidb_broadcast_join_threshold_size-new-in-v50) variable controls whether to use the Broadcast Hash Join algorithm. If the table size (unit: byte) is smaller than the value of this variable, the Broadcast Hash Join algorithm is used. Otherwise, the Shuffled Hash Join algorithm is used. @@ -244,7 +243,7 @@ For `Join` operations with small tables, the Broadcast Hash Join algorithm can a set @@tidb_broadcast_join_threshold_count = 100000; ``` -The following example shows the query result before and after `tidb_broadcast_join_threshold_size` is re-configured. Before the re-configuration, the `ExchangeType` of `ExchangeSender_29` is `HashPartition`. After the value of this variable chages to `10000000`, the `ExchangeType` of `ExchangeSender_29` changes to `Broadcast`. +The following example shows the query result before and after `tidb_broadcast_join_threshold_size` is re-configured. Before the re-configuration, the `ExchangeType` of `ExchangeSender_29` is `HashPartition`. After the value of this variable changes to `10000000`, the `ExchangeType` of `ExchangeSender_29` changes to `Broadcast`. Before `tidb_broadcast_join_threshold_size` is re-configured: @@ -305,7 +304,7 @@ The [`tidb_max_tiflash_threads`](/system-variables.md#tidb_max_tiflash_threads-n set @@tidb_max_tiflash_threads = 20; ``` -The following example shows the query result before and after `tidb_max_tiflash_threads` is re-configured. Before the re-configuration, the execution concurrency of all TiFlash operators is 24. After the value of this variable changes to `20`, the concurrency becomes 60. +The following example shows the query result before and after `tidb_max_tiflash_threads` is re-configured. Before `tidb_max_tiflash_threads` is set, the concurrency of request execution for a single TiFlash instance is 8 threads. Since the cluster has a total of 3 TiFlash instances, the total number of threads for request execution on all TiFlash instances is 24 (8 × 3). After `tidb_max_tiflash_threads` is set to `20`, the total number of threads for request execution on all TiFlash instances is 60 (20 × 3). Before `tidb_max_tiflash_threads` is re-configured: diff --git a/tiflash/use-fastscan.md b/tiflash/use-fastscan.md index 7683dc8026d1b..4108d81468bfc 100644 --- a/tiflash/use-fastscan.md +++ b/tiflash/use-fastscan.md @@ -1,7 +1,6 @@ --- title: Use FastScan summary: Introduces a way to speed up querying in OLAP scenarios by using FastScan. -aliases: ['/tidb/dev/sql-statement-set-tiflash-mode/','/tidb/dev/dev-guide-use-fastscan/'] --- # Use FastScan diff --git a/tiflash/use-tidb-to-read-tiflash.md b/tiflash/use-tidb-to-read-tiflash.md index 325f1e25d5f74..afba02d280ae0 100644 --- a/tiflash/use-tidb-to-read-tiflash.md +++ b/tiflash/use-tidb-to-read-tiflash.md @@ -143,4 +143,7 @@ In the above three ways of reading TiFlash replicas, engine isolation specifies > **Note:** > -> Before v4.0.3, the behavior of reading from TiFlash replica in a non-read-only SQL statement (for example, `INSERT INTO ... SELECT`, `SELECT ... FOR UPDATE`, `UPDATE ...`, `DELETE ...`) is undefined. In v4.0.3 and later versions, internally TiDB ignores the TiFlash replica for a non-read-only SQL statement to guarantee the data correctness. That is, for [smart selection](#smart-selection), TiDB automatically selects the non-TiFlash replica; for [engine isolation](#engine-isolation) that specifies TiFlash replica **only**, TiDB reports an error; and for [manual hint](#manual-hint), TiDB ignores the hint. +> - Before v4.0.3, the behavior of reading from TiFlash replica in a non-read-only SQL statement (for example, `INSERT INTO ... SELECT`, `SELECT ... FOR UPDATE`, `UPDATE ...`, `DELETE ...`) is undefined. +> - For versions from v4.0.3 to v6.2.0, internally TiDB ignores the TiFlash replica for a non-read-only SQL statement to guarantee the data correctness. That is, for [smart selection](#smart-selection), TiDB automatically selects the non-TiFlash replica; for [engine isolation](#engine-isolation) that specifies TiFlash replica **only**, TiDB reports an error; and for [manual hint](#manual-hint), TiDB ignores the hint. +> - For versions from v6.3.0 to v7.0.0, if the TiFlash replica is enabled, you can use the [`tidb_enable_tiflash_read_for_write_stmt`](/system-variables.md#tidb_enable_tiflash_read_for_write_stmt-new-in-v630) variable to control whether TiDB uses the TiFlash replica for a non-read-only SQL statement. +> - Starting from v7.1.0, if the TiFlash replica is enabled and the [SQL Mode](/sql-mode.md) of the current session is not strict (which means the `sql_mode` value does not contain `STRICT_TRANS_TABLES` or `STRICT_ALL_TABLES`), TiDB automatically decides whether to use the TiFlash replica for a non-read-only SQL statement based on cost estimation. diff --git a/tiflash/use-tiflash-mpp-mode.md b/tiflash/use-tiflash-mpp-mode.md index a2675e0241e48..7f09ca3be0807 100644 --- a/tiflash/use-tiflash-mpp-mode.md +++ b/tiflash/use-tiflash-mpp-mode.md @@ -95,19 +95,19 @@ The following statement takes the table structure in the TPC-H test set as an ex ```sql explain select count(*) from customer c join nation n on c.c_nationkey=n.n_nationkey; -+------------------------------------------+------------+-------------------+---------------+----------------------------------------------------------------------------+ -| id | estRows | task | access object | operator info | -+------------------------------------------+------------+-------------------+---------------+----------------------------------------------------------------------------+ -| HashAgg_23 | 1.00 | root | | funcs:count(Column#16)->Column#15 | -| └─TableReader_25 | 1.00 | root | | data:ExchangeSender_24 | -| └─ExchangeSender_24 | 1.00 | batchCop[tiflash] | | ExchangeType: PassThrough | -| └─HashAgg_12 | 1.00 | batchCop[tiflash] | | funcs:count(1)->Column#16 | -| └─HashJoin_17 | 3000000.00 | batchCop[tiflash] | | inner join, equal:[eq(tpch.nation.n_nationkey, tpch.customer.c_nationkey)] | -| ├─ExchangeReceiver_21(Build) | 25.00 | batchCop[tiflash] | | | -| │ └─ExchangeSender_20 | 25.00 | batchCop[tiflash] | | ExchangeType: Broadcast | -| │ └─TableFullScan_18 | 25.00 | batchCop[tiflash] | table:n | keep order:false | -| └─TableFullScan_22(Probe) | 3000000.00 | batchCop[tiflash] | table:c | keep order:false | -+------------------------------------------+------------+-------------------+---------------+----------------------------------------------------------------------------+ ++------------------------------------------+------------+--------------+---------------+----------------------------------------------------------------------------+ +| id | estRows | task | access object | operator info | ++------------------------------------------+------------+--------------+---------------+----------------------------------------------------------------------------+ +| HashAgg_23 | 1.00 | root | | funcs:count(Column#16)->Column#15 | +| └─TableReader_25 | 1.00 | root | | data:ExchangeSender_24 | +| └─ExchangeSender_24 | 1.00 | mpp[tiflash] | | ExchangeType: PassThrough | +| └─HashAgg_12 | 1.00 | mpp[tiflash] | | funcs:count(1)->Column#16 | +| └─HashJoin_17 | 3000000.00 | mpp[tiflash] | | inner join, equal:[eq(tpch.nation.n_nationkey, tpch.customer.c_nationkey)] | +| ├─ExchangeReceiver_21(Build) | 25.00 | mpp[tiflash] | | | +| │ └─ExchangeSender_20 | 25.00 | mpp[tiflash] | | ExchangeType: Broadcast | +| │ └─TableFullScan_18 | 25.00 | mpp[tiflash] | table:n | keep order:false | +| └─TableFullScan_22(Probe) | 3000000.00 | mpp[tiflash] | table:c | keep order:false | ++------------------------------------------+------------+--------------+---------------+----------------------------------------------------------------------------+ 9 rows in set (0.00 sec) ``` diff --git a/tikv-configuration-file.md b/tikv-configuration-file.md index 8a6244b73cb30..e682ca4a2ef43 100644 --- a/tikv-configuration-file.md +++ b/tikv-configuration-file.md @@ -1,14 +1,13 @@ --- title: TiKV Configuration File summary: Learn the TiKV configuration file. -aliases: ['/docs/dev/tikv-configuration-file/','/docs/dev/reference/configuration/tikv-server/configuration-file/'] --- # TiKV Configuration File -The TiKV configuration file supports more options than command-line parameters. You can find the default configuration file in [etc/config-template.toml](https://github.com/tikv/tikv/blob/master/etc/config-template.toml) and rename it to `config.toml`. +The TiKV configuration file supports more options than command-line parameters. You can find the default configuration file in [etc/config-template.toml](https://github.com/tikv/tikv/blob/release-7.5/etc/config-template.toml) and rename it to `config.toml`. This document only describes the parameters that are not included in command-line parameters. For more details, see [command-line parameter](/command-line-flags-for-tikv-configuration.md). @@ -37,7 +36,7 @@ This document only describes the parameters that are not included in command-lin ### `slow-log-threshold` -+ The threshold for outputing slow logs. If the processing time is longer than this threshold, slow logs are output. ++ The threshold for outputting slow logs. If the processing time is longer than this threshold, slow logs are output. + Default value: `"1s"` ### `memory-usage-limit` @@ -105,13 +104,6 @@ This document only describes the parameters that are not included in command-lin + If the configuration item is set to a value other than `0`, TiKV keeps at most the number of old log files specified by `max-backups`. For example, if the value is set to `7`, TiKV keeps up to 7 old log files. + Default value: `0` -### `pd.enable-forwarding` New in v5.0.0 - -+ Controls whether the PD client in TiKV forwards requests to the leader via the followers in the case of possible network isolation. -+ Default value: `false` -+ If the environment might have isolated network, enabling this parameter can reduce the window of service unavailability. -+ If you cannot accurately determine whether isolation, network interruption, or downtime has occurred, using this mechanism has the risk of misjudgment and causes reduced availability and performance. If network failure has never occurred, it is not recommended to enable this parameter. - ## server + Configuration items related to the server. @@ -146,8 +138,13 @@ This document only describes the parameters that are not included in command-lin ### `grpc-compression-type` -+ The compression algorithm for gRPC messages ++ The compression algorithm for gRPC messages. It affects gRPC messages between TiKV nodes. Starting from v6.5.11, v7.1.6, and v7.5.3, it also affects gRPC response messages sent from TiKV to TiDB. + Optional values: `"none"`, `"deflate"`, `"gzip"` + + > **Note:** + > + > TiDB does not support `"deflate"`. Therefore, if you want to compress gRPC response messages sent from TiKV to TiDB, set this configuration item to `"gzip"`. + + Default value: `"none"` ### `grpc-concurrency` @@ -170,7 +167,7 @@ This document only describes the parameters that are not included in command-lin ### `grpc-raft-conn-num` -+ The maximum number of links among TiKV nodes for Raft communication ++ The maximum number of connections between TiKV nodes for Raft communication + Default value: `1` + Minimum value: `1` @@ -184,9 +181,9 @@ This document only describes the parameters that are not included in command-lin ### `grpc-stream-initial-window-size` + The window size of the gRPC stream -+ Default value: `2MB` -+ Unit: KB|MB|GB -+ Minimum value: `"1KB"` ++ Default value: `2MiB` ++ Unit: KiB|MiB|GiB ++ Minimum value: `"1KiB"` ### `grpc-keepalive-time` @@ -227,9 +224,9 @@ This document only describes the parameters that are not included in command-lin ### `snap-io-max-bytes-per-sec` + The maximum allowable disk bandwidth when processing snapshots -+ Default value: `"100MB"` -+ Unit: KB|MB|GB -+ Minimum value: `"1KB"` ++ Default value: `"100MiB"` ++ Unit: KiB|MiB|GiB ++ Minimum value: `"1KiB"` ### `enable-request-batch` @@ -255,7 +252,7 @@ This document only describes the parameters that are not included in command-lin ### `raft-client-queue-size` + Specifies the queue size of the Raft messages in TiKV. If too many messages not sent in time result in a full buffer, or messages discarded, you can specify a greater value to improve system stability. -+ Default value: `8192` ++ Default value: `16384`. Before v7.5.5, the default value is `8192`. ### `simplify-metrics` New in v6.2.0 @@ -290,9 +287,9 @@ Configuration items related to the single thread pool serving read requests. Thi + The stack size of the threads in the unified thread pool + Type: Integer + Unit -+ Default value: `"10MB"` -+ Unit: KB|MB|GB -+ Minimum value: `"2MB"` ++ Default value: `"10MiB"` ++ Unit: KiB|MiB|GiB ++ Minimum value: `"2MiB"` + Maximum value: The number of Kbytes output in the result of the `ulimit -sH` command executed in the system. ### `max-tasks-per-worker` @@ -355,9 +352,9 @@ Configuration items related to storage thread pool. + The stack size of threads in the Storage read thread pool + Type: Integer + Unit -+ Default value: `"10MB"` -+ Unit: KB|MB|GB -+ Minimum value: `"2MB"` ++ Default value: `"10MiB"` ++ Unit: KiB|MiB|GiB ++ Minimum value: `"2MiB"` + Maximum value: The number of Kbytes output in the result of the `ulimit -sH` command executed in the system. ## `readpool.coprocessor` @@ -409,9 +406,9 @@ Configuration items related to the Coprocessor thread pool. + The stack size of the thread in the Coprocessor thread pool + Type: Integer + Unit -+ Default value: `"10MB"` -+ Unit: KB|MB|GB -+ Minimum value: `"2MB"` ++ Default value: `"10MiB"` ++ Unit: KiB|MiB|GiB ++ Minimum value: `"2MiB"` + Maximum value: The number of Kbytes output in the result of the `ulimit -sH` command executed in the system. ## storage @@ -451,8 +448,8 @@ Configuration items related to storage. ### `scheduler-pending-write-threshold` + The maximum size of the write queue. A `Server Is Busy` error is returned for a new write to TiKV when this value is exceeded. -+ Default value: `"100MB"` -+ Unit: MB|GB ++ Default value: `"100MiB"` ++ Unit: MiB|GiB ### `enable-async-apply-prewrite` @@ -463,18 +460,18 @@ Configuration items related to storage. + When TiKV is started, some space is reserved on the disk as disk protection. When the remaining disk space is less than the reserved space, TiKV restricts some write operations. The reserved space is divided into two parts: 80% of the reserved space is used as the extra disk space required for operations when the disk space is insufficient, and the other 20% is used to store the temporary file. In the process of reclaiming space, if the storage is exhausted by using too much extra disk space, this temporary file serves as the last protection for restoring services. + The name of the temporary file is `space_placeholder_file`, located in the `storage.data-dir` directory. When TiKV goes offline because its disk space ran out, if you restart TiKV, the temporary file is automatically deleted and TiKV tries to reclaim the space. -+ When the remaining space is insufficient, TiKV does not create the temporary file. The effectiveness of the protection is related to the size of the reserved space. The size of the reserved space is the larger value between 5% of the disk capacity and this configuration value. When the value of this configuration item is `"0MB"`, TiKV disables this disk protection feature. -+ Default value: `"5GB"` -+ Unit: MB|GB ++ When the remaining space is insufficient, TiKV does not create the temporary file. The effectiveness of the protection is related to the size of the reserved space. The size of the reserved space is the larger value between 5% of the disk capacity and this configuration value. If this configuration item is set to `0`, or to a zero value with any supported unit (for example, `0KiB`, `0MiB`, or `0GiB`), TiKV disables this disk protection feature. ++ Default value: `"5GiB"` ++ Unit: B|KB|KiB|MB|MiB|GB|GiB|TB|TiB|PB|PiB ### `enable-ttl` > **Warning:** > > - Set `enable-ttl` to `true` or `false` **ONLY WHEN** deploying a new TiKV cluster. **DO NOT** modify the value of this configuration item in an existing TiKV cluster. TiKV clusters with different `enable-ttl` values use different data formats. Therefore, if you modify the value of this item in an existing TiKV cluster, the cluster will store data in different formats, which causes the "can't enable TTL on a non-ttl" error when you restart the TiKV cluster. -> - Use `enable-ttl` **ONLY IN** a TiKV cluster. **DO NOT** use this configuration item in a cluster that has TiDB nodes (which means setting `enable-ttl` to `true` in such clusters). Otherwise, critical issues such as data corruption and the upgrade failure of TiDB clusters will occur. +> - Use `enable-ttl` **ONLY IN** a TiKV cluster. **DO NOT** use this configuration item in a cluster that has TiDB nodes (which means setting `enable-ttl` to `true` in such clusters) unless `storage.api-version = 2` is configured. Otherwise, critical issues such as data corruption and the upgrade failure of TiDB clusters will occur. -+ TTL is short for "Time to live". If this item is enabled, TiKV automatically deletes data that reaches its TTL. To set the value of TTL, you need to specify it in the requests when writing data via the client. If the TTL is not specified, it means that TiKV does not automatically delete the corresponding data. ++ [TTL](/time-to-live.md) is short for "Time to live". If this item is enabled, TiKV automatically deletes data that reaches its TTL. To set the value of TTL, you need to specify it in the requests when writing data via the client. If the TTL is not specified, it means that TiKV does not automatically delete the corresponding data. + Default value: `false` ### `ttl-check-poll-interval` @@ -496,9 +493,9 @@ Configuration items related to storage. + Value options: + `1`: Uses API V1, does not encode the data passed from the client, and stores data as it is. In versions earlier than v6.1.0, TiKV uses API V1 by default. + `2`: Uses API V2: - + The data is stored in the Multi-Version Concurrency Control (MVCC) format, where the timestamp is obtained from PD (which is TSO) by tikv-server. + + The data is stored in the [Multi-Version Concurrency Control (MVCC)](/glossary.md#multi-version-concurrency-control-mvcc) format, where the timestamp is obtained from PD (which is TSO) by tikv-server. + Data is scoped according to different usage and API V2 supports co-existence of TiDB, Transactional KV, and RawKV applications in a single cluster. - + When API V2 is used, you are expected to set `storage.enable-ttl = true` at the same time. Because API V2 supports the TTL feature, you must turn on `enable-ttl` explicitly. Otherwise, it will be in conflict because `storage.enable-ttl` defaults to `false`. + + When API V2 is used, you are expected to set `storage.enable-ttl = true` at the same time. Because API V2 supports the TTL feature, you must turn on [`enable-ttl`](#enable-ttl) explicitly. Otherwise, it will be in conflict because `storage.enable-ttl` defaults to `false`. + When API V2 is enabled, you need to deploy at least one tidb-server instance to reclaim obsolete data. This tidb-server instance can provide read and write services at the same time. To ensure high availability, you can deploy multiple tidb-server instances. + Client support is required for API V2. For details, see the corresponding instruction of the client for the API V2. + Since v6.2.0, Change Data Capture (CDC) for RawKV is supported. Refer to [RawKV CDC](https://tikv.org/docs/latest/concepts/explore-tikv-features/cdc/cdc). @@ -521,7 +518,7 @@ Configuration items related to the sharing of block cache among multiple RocksDB + When `storage.engine="raft-kv"`, the default value is 45% of the size of total system memory. + When `storage.engine="partitioned-raft-kv"`, the default value is 30% of the size of total system memory. -+ Unit: KB|MB|GB ++ Unit: KiB|MiB|GiB ## storage.flow-control @@ -545,12 +542,12 @@ Configuration items related to the flow control mechanism in TiKV. This mechanis ### `soft-pending-compaction-bytes-limit` + When the pending compaction bytes in KvDB reach this threshold, the flow control mechanism starts to reject some write requests and reports the `ServerIsBusy` error. When `enable` is set to `true`, this configuration item overrides `rocksdb.(defaultcf|writecf|lockcf).soft-pending-compaction-bytes-limit`. -+ Default value: `"192GB"` ++ Default value: `"192GiB"` ### `hard-pending-compaction-bytes-limit` + When the pending compaction bytes in KvDB reach this threshold, the flow control mechanism rejects all write requests and reports the `ServerIsBusy` error. When `enable` is set to `true`, this configuration item overrides `rocksdb.(defaultcf|writecf|lockcf).hard-pending-compaction-bytes-limit`. -+ Default value: `"1024GB"` ++ Default value: `"1024GiB"` ## storage.io-rate-limit @@ -559,7 +556,7 @@ Configuration items related to the I/O rate limiter. ### `max-bytes-per-sec` + Limits the maximum I/O bytes that a server can write to or read from the disk (determined by the `mode` configuration item below) in one second. When this limit is reached, TiKV prefers throttling background operations over foreground ones. The value of this configuration item should be set to the disk's optimal I/O bandwidth, for example, the maximum I/O bandwidth specified by your cloud disk vendor. When this configuration value is set to zero, disk I/O operations are not limited. -+ Default value: `"0MB"` ++ Default value: `"0MiB"` ### `mode` @@ -569,6 +566,13 @@ Configuration items related to the I/O rate limiter. ## pd +### `enable-forwarding` New in v5.0.0 + ++ Controls whether the PD client in TiKV forwards requests to the leader via the followers in the case of possible network isolation. ++ Default value: `false` ++ If the environment might have isolated network, enabling this parameter can reduce the window of service unavailability. ++ If you cannot accurately determine whether isolation, network interruption, or downtime has occurred, using this mechanism has the risk of misjudgment and causes reduced availability and performance. If network failure has never occurred, it is not recommended to enable this parameter. + ### `endpoints` + The endpoints of PD. When multiple endpoints are specified, you need to separate them using commas. @@ -604,7 +608,7 @@ Configuration items related to Raftstore. + The storage capacity, which is the maximum size allowed to store data. If `capacity` is left unspecified, the capacity of the current disk prevails. To deploy multiple TiKV instances on the same physical disk, add this parameter to the TiKV configuration. For details, see [Key parameters of the hybrid deployment](/hybrid-deployment-topology.md#key-parameters). + Default value: `0` -+ Unit: KB|MB|GB ++ Unit: KiB|MiB|GiB ### `raftdb-path` @@ -663,22 +667,14 @@ Configuration items related to Raftstore. ### `raft-max-size-per-msg` -> **Note:** -> -> This configuration item cannot be queried via SQL statements but can be configured in the configuration file. - + The soft limit on the size of a single message packet -+ Default value: `"1MB"` ++ Default value: `"1MiB"` + Minimum value: greater than `0` -+ Maximum value: `3GB` -+ Unit: KB|MB|GB ++ Maximum value: `3GiB` ++ Unit: KiB|MiB|GiB ### `raft-max-inflight-msgs` -> **Note:** -> -> This configuration item cannot be queried via SQL statements but can be configured in the configuration file. - + The number of Raft logs to be confirmed. If this number is exceeded, the Raft state machine slows down log sending. + Default value: `256` + Minimum value: greater than `0` @@ -687,9 +683,9 @@ Configuration items related to Raftstore. ### `raft-entry-max-size` + The hard limit on the maximum size of a single log -+ Default value: `"8MB"` ++ Default value: `"8MiB"` + Minimum value: `0` -+ Unit: MB|GB ++ Unit: MiB|GiB ### `raft-log-compact-sync-interval` New in v5.3 @@ -712,7 +708,7 @@ Configuration items related to Raftstore. ### `raft-log-gc-count-limit` + The hard limit on the allowable number of residual Raft logs -+ Default value: the log number that can be accommodated in the 3/4 Region size (calculated as 1MB for each log) ++ Default value: the number of logs that can fit into three-fourths of the Region size, calculated assuming each log is 1 KiB + Minimum value: `0` ### `raft-log-gc-size-limit` @@ -757,12 +753,20 @@ Configuration items related to Raftstore. ### `region-compact-check-interval` +> **Warning:** +> +> Starting from v7.5.7, this configuration item is deprecated and replaced by [`gc.auto-compaction.check-interval`](#check-interval-new-in-v757). + + The time interval at which to check whether it is necessary to manually trigger RocksDB compaction. `0` means that this feature is disabled. + Default value: `"5m"` + Minimum value: `0` ### `region-compact-check-step` +> **Warning:** +> +> Starting from v7.5.7, this configuration item is deprecated. + + The number of Regions checked at one time for each round of manual compaction + Default value: @@ -772,12 +776,20 @@ Configuration items related to Raftstore. ### `region-compact-min-tombstones` +> **Warning:** +> +> Starting from v7.5.7, this configuration item is deprecated and replaced by [`gc.auto-compaction.tombstone-num-threshold`](#tombstone-num-threshold-new-in-v757). + + The number of tombstones required to trigger RocksDB compaction + Default value: `10000` + Minimum value: `0` ### `region-compact-tombstones-percent` +> **Warning:** +> +> Starting from v7.5.7, this configuration item is deprecated and replaced by [`gc.auto-compaction.tombstone-percent-threshold`](#tombstone-percent-threshold-new-in-v757). + + The proportion of tombstone required to trigger RocksDB compaction + Default value: `30` + Minimum value: `1` @@ -785,13 +797,21 @@ Configuration items related to Raftstore. ### `region-compact-min-redundant-rows` New in v7.1.0 -+ The number of redundant MVCC rows required to trigger RocksDB compaction. This configuration only takes effect for Partitioned Raft KV (`storage.engine="partitioned-raft-kv"`). +> **Warning:** +> +> Starting from v7.5.7, this configuration item is deprecated and replaced by [`gc.auto-compaction.redundant-rows-threshold`](#redundant-rows-threshold-new-in-v757). + ++ The number of redundant MVCC rows required to trigger RocksDB compaction. + Default value: `50000` + Minimum value: `0` ### `region-compact-redundant-rows-percent` New in v7.1.0 -+ The percentage of redundant MVCC rows required to trigger RocksDB compaction. This configuration only takes effect for Partitioned Raft KV (`storage.engine="partitioned-raft-kv"`). +> **Warning:** +> +> Starting from v7.5.7, this configuration item is deprecated and replaced by [`gc.auto-compaction.redundant-rows-percent-threshold`](#redundant-rows-percent-threshold-new-in-v757). + ++ The percentage of redundant MVCC rows required to trigger RocksDB compaction. + Default value: `20` + Minimum value: `1` + Maximum value: `100` @@ -845,9 +865,9 @@ Configuration items related to Raftstore. ### `lock-cf-compact-bytes-threshold` + The size out of which TiKV triggers a manual compaction for the Lock Column Family -+ Default value: `"256MB"` ++ Default value: `"256MiB"` + Minimum value: `0` -+ Unit: MB ++ Unit: MiB ### `notify-capacity` @@ -900,9 +920,9 @@ Configuration items related to Raftstore. ### `snap-apply-batch-size` + The memory cache size required when the imported snapshot file is written into the disk -+ Default value: `"10MB"` ++ Default value: `"10MiB"` + Minimum value: `0` -+ Unit: MB ++ Unit: MiB ### `consistency-check-interval` @@ -1008,13 +1028,13 @@ Configuration items related to Raftstore. + At a certain interval, TiKV inspects the latency of the Raftstore component. This parameter specifies the interval of the inspection. If the latency exceeds this value, this inspection is marked as timeout. + Judges whether the TiKV node is slow based on the ratio of timeout inspection. -+ Default value: `"500ms"` ++ Default value: `"100ms"` + Minimum value: `"1ms"` ### `raft-write-size-limit` New in v5.3.0 + Determines the threshold at which Raft data is written into the disk. If the data size is larger than the value of this configuration item, the data is written to the disk. When the value of `store-io-pool-size` is `0`, this configuration item does not take effect. -+ Default value: `1MB` ++ Default value: `1MiB` + Minimum value: `0` ### `report-min-resolved-ts-interval` New in v6.0.0 @@ -1024,6 +1044,33 @@ Configuration items related to Raftstore. + Minimum value: `0` + Unit: second +### `evict-cache-on-memory-ratio` New in v7.5.0 + ++ When the memory usage of TiKV exceeds 90% of the system available memory, and the memory occupied by Raft entry cache exceeds the used memory * `evict-cache-on-memory-ratio`, TiKV evicts the Raft entry cache. ++ If this value is set to `0`, it means that this feature is disabled. ++ Default value: `0.1` ++ Minimum value: `0` + +### `follower-read-max-log-gap` New in v7.4.0 + ++ The maximum number of logs a follower is allowed to lag behind when processing read requests. If this limit is exceeded, the read request is rejected. ++ Default value: `100` + +### `request-voter-replicated-index-interval` New in v6.6.0 + ++ Controls the interval at which the Witness node periodically retrieves the replicated Raft log position from voter nodes. ++ Default value: `5m`, which means 5 minutes. + +### `slow-trend-unsensitive-cause` New in v6.6.0 + ++ When TiKV uses the SlowTrend detection algorithm, this configuration item controls the sensitivity of latency detection. A higher value indicates lower sensitivity. ++ Default value: `10` + +### `slow-trend-unsensitive-result` New in v6.6.0 + ++ When TiKV uses the SlowTrend detection algorithm, this configuration item controls the sensitivity of QPS detection. A higher value indicates lower sensitivity. ++ Default value: `0.5` + ## coprocessor Configuration items related to Coprocessor. @@ -1132,9 +1179,9 @@ Configuration items related to RocksDB ### `max-manifest-file-size` + The maximum size of a RocksDB Manifest file -+ Default value: `"128MB"` ++ Default value: `"128MiB"` + Minimum value: `0` -+ Unit: B|KB|MB|GB ++ Unit: B|KiB|MiB|GiB ### `create-if-missing` @@ -1153,8 +1200,8 @@ Configuration items related to RocksDB ### `wal-dir` -+ The directory in which WAL files are stored -+ Default value: `"/tmp/tikv/store"` ++ The directory in which WAL files are stored. If not specified, the WAL files will be stored in the same directory as the data. ++ Default value: `""` ### `wal-ttl-seconds` @@ -1168,14 +1215,14 @@ Configuration items related to RocksDB + The size limit of the archived WAL files. When the value is exceeded, the system deletes these files. + Default value: `0` + Minimum value: `0` -+ Unit: B|KB|MB|GB ++ Unit: B|KiB|MiB|GiB ### `max-total-wal-size` + The maximum RocksDB WAL size in total, which is the size of `*.log` files in the `data-dir`. + Default value: - + When `storage.engine="raft-kv"`, the default value is `"4GB"`. + + When `storage.engine="raft-kv"`, the default value is `"4GiB"`. + When `storage.engine="partitioned-raft-kv"`, the default value is `1`. ### `stats-dump-period` @@ -1188,17 +1235,17 @@ Configuration items related to RocksDB ### `compaction-readahead-size` -+ Enables the readahead feature during RocksDB compaction and specifies the size of readahead data. If you are using mechanical disks, it is recommended to set the value to 2MB at least. ++ Enables the readahead feature during RocksDB compaction and specifies the size of readahead data. If you are using mechanical disks, it is recommended to set the value to 2MiB at least. + Default value: `0` + Minimum value: `0` -+ Unit: B|KB|MB|GB ++ Unit: B|KiB|MiB|GiB ### `writable-file-max-buffer-size` + The maximum buffer size used in WritableFileWrite -+ Default value: `"1MB"` ++ Default value: `"1MiB"` + Minimum value: `0` -+ Unit: B|KB|MB|GB ++ Unit: B|KiB|MiB|GiB ### `use-direct-io-for-flush-and-compaction` @@ -1208,9 +1255,9 @@ Configuration items related to RocksDB ### `rate-bytes-per-sec` + The maximum rate permitted by RocksDB's compaction rate limiter -+ Default value: `10GB` ++ Default value: `10GiB` + Minimum value: `0` -+ Unit: B|KB|MB|GB ++ Unit: B|KiB|MiB|GiB ### `rate-limiter-refill-period` @@ -1236,31 +1283,43 @@ Configuration items related to RocksDB ### `bytes-per-sync` + The rate at which OS incrementally synchronizes files to disk while these files are being written asynchronously -+ Default value: `"1MB"` ++ Default value: `"1MiB"` + Minimum value: `0` -+ Unit: B|KB|MB|GB ++ Unit: B|KiB|MiB|GiB ### `wal-bytes-per-sync` + The rate at which OS incrementally synchronizes WAL files to disk while the WAL files are being written -+ Default value: `"512KB"` ++ Default value: `"512KiB"` + Minimum value: `0` -+ Unit: B|KB|MB|GB ++ Unit: B|KiB|MiB|GiB ### `info-log-max-size` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated, and its function is replaced by the configuration item [`log.file.max-size`](#max-size-new-in-v540). + + The maximum size of Info log -+ Default value: `"1GB"` ++ Default value: `"1GiB"` + Minimum value: `0` -+ Unit: B|KB|MB|GB ++ Unit: B|KiB|MiB|GiB ### `info-log-roll-time` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated. TiKV no longer supports automatic log splitting based on time. Instead, you can use the configuration item [`log.file.max-size`](#max-size-new-in-v540) to set the threshold for automatic log splitting based on file size. + + The time interval at which Info logs are truncated. If the value is `0s`, logs are not truncated. + Default value: `"0s"` ### `info-log-keep-log-file-num` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated, and its function is replaced by the configuration item [`log.file.max-backups`](#max-backups-new-in-v540). + + The maximum number of kept log files + Default value: `10` + Minimum value: `0` @@ -1272,6 +1331,10 @@ Configuration items related to RocksDB ### `info-log-level` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated, and its function is replaced by the configuration item [`log.level`](#level-new-in-v540). + + Log levels of RocksDB + Default value: `"info"` @@ -1302,12 +1365,29 @@ Configuration items related to RocksDB + Unit: KiB|MiB|GiB +### `track-and-verify-wals-in-manifest` New in v6.5.9, v7.1.5, and v7.5.2 + ++ Controls whether to record information about Write Ahead Log (WAL) files in the RocksDB MANIFEST file and whether to verify the integrity of WAL files during startup. For more information, see RocksDB [Track WAL in MANIFEST](https://github.com/facebook/rocksdb/wiki/Track-WAL-in-MANIFEST). ++ Default value: `false` ++ Value options: + + `true`: records information about WAL files in the MANIFEST file and verifies the integrity of WAL files during startup. + + `false`: does not record information about WAL files in the MANIFEST file and does not verify the integrity of WAL files during startup. + +### `enable-multi-batch-write` New in v6.2.0 + ++ Controls whether to enable RocksDB write optimization, allowing the contents of WriteBatch to be written concurrently to the memtable, reducing write latency. ++ Default value: None. However, it is enabled by default unless explicitly set to `false` or if `rocksdb.enable-pipelined-write` or `rocksdb.enable-unordered-write` is enabled. + ## rocksdb.titan Configuration items related to Titan. ### `enabled` +> **Warning** +> +> When disabling Titan for TiDB versions earlier than v8.5.0, it is not recommended to modify this configuration item to `false`, as this might cause TiKV to crash. To disable Titan, refer to the steps in [Disable Titan](/storage-engine/titan-configuration.md#disable-titan). + + Enables or disables Titan + Default value: `false` @@ -1327,17 +1407,17 @@ Configuration items related to Titan. + Default value: `4` + Minimum value: `1` -## rocksdb.defaultcf | rocksdb.writecf | rocksdb.lockcf +## rocksdb.defaultcf | rocksdb.writecf | rocksdb.lockcf | rocksdb.raftcf Configuration items related to `rocksdb.defaultcf`, `rocksdb.writecf`, and `rocksdb.lockcf`. ### `block-size` + The default size of a RocksDB block -+ Default value for `defaultcf` and `writecf`: `"32KB"` -+ Default value for `lockcf`: `"16KB"` -+ Minimum value: `"1KB"` -+ Unit: KB|MB|GB ++ Default value for `defaultcf` and `writecf`: `"32KiB"` ++ Default value for `lockcf`: `"16KiB"` ++ Minimum value: `"1KiB"` ++ Unit: KiB|MiB|GiB ### `block-cache-size` @@ -1350,7 +1430,7 @@ Configuration items related to `rocksdb.defaultcf`, `rocksdb.writecf`, and `rock + Default value for `writecf`: `Total machine memory * 15%` + Default value for `lockcf`: `Total machine memory * 2%` + Minimum value: `0` -+ Unit: KB|MB|GB ++ Unit: KiB|MiB|GiB ### `disable-block-cache` @@ -1431,12 +1511,12 @@ Configuration items related to `rocksdb.defaultcf`, `rocksdb.writecf`, and `rock ### `write-buffer-size` + Memtable size -+ Default value for `defaultcf` and `writecf`: `"128MB"` ++ Default value for `defaultcf` and `writecf`: `"128MiB"` + Default value for `lockcf`: - + When `storage.engine="raft-kv"`, the default value is `"32MB"`. - + When `storage.engine="partitioned-raft-kv"`, the default value is `"4MB"`. + + When `storage.engine="raft-kv"`, the default value is `"32MiB"`. + + When `storage.engine="partitioned-raft-kv"`, the default value is `"4MiB"`. + Minimum value: `0` -+ Unit: KB|MB|GB ++ Unit: KiB|MiB|GiB ### `max-write-buffer-number` @@ -1453,18 +1533,18 @@ Configuration items related to `rocksdb.defaultcf`, `rocksdb.writecf`, and `rock ### `max-bytes-for-level-base` + The maximum number of bytes at base level (level-1). Generally, it is set to 4 times the size of a memtable. When the level-1 data size reaches the limit value of `max-bytes-for-level-base`, the SST files of level-1 and their overlapping SST files of level-2 will be compacted. -+ Default value for `defaultcf` and `writecf`: `"512MB"` -+ Default value for `lockcf`: `"128MB"` ++ Default value for `defaultcf` and `writecf`: `"512MiB"` ++ Default value for `lockcf`: `"128MiB"` + Minimum value: `0` -+ Unit: KB|MB|GB -+ It is recommended that the value of `max-bytes-for-level-base` is set approximately equal to the data volume in L0 to reduce unnecessary compaction. For example, if the compression method is "no:no:lz4:lz4:lz4:lz4:lz4", the value of `max-bytes-for-level-base` should be `write-buffer-size * 4`, because there is no compression of L0 and L1 and the trigger condition of compaction for L0 is that the number of the SST files reaches 4 (the default value). When L0 and L1 both adopt compaction, you need to analyze RocksDB logs to understand the size of an SST file compressed from a memtable. For example, if the file size is 32 MB, it is recommended to set the value of `max-bytes-for-level-base` to 128 MB (`32 MB * 4`). ++ Unit: KiB|MiB|GiB ++ It is recommended that the value of `max-bytes-for-level-base` is set approximately equal to the data volume in L0 to reduce unnecessary compaction. For example, if the compression method is "no:no:lz4:lz4:lz4:lz4:lz4", the value of `max-bytes-for-level-base` should be `write-buffer-size * 4`, because there is no compression of L0 and L1 and the trigger condition of compaction for L0 is that the number of the SST files reaches 4 (the default value). When L0 and L1 both adopt compaction, you need to analyze RocksDB logs to understand the size of an SST file compressed from a memtable. For example, if the file size is 32 MiB, it is recommended to set the value of `max-bytes-for-level-base` to 128 MiB (`32 MiB * 4`). ### `target-file-size-base` + The size of the target file at base level. This value is overridden by `compaction-guard-max-output-file-size` when the `enable-compaction-guard` value is `true`. -+ Default value: `"8MB"` ++ Default value: `"8MiB"` + Minimum value: `0` -+ Unit: KB|MB|GB ++ Unit: KiB|MiB|GiB ### `level0-file-num-compaction-trigger` @@ -1488,9 +1568,9 @@ Configuration items related to `rocksdb.defaultcf`, `rocksdb.writecf`, and `rock ### `max-compaction-bytes` + The maximum number of bytes written into disk per compaction -+ Default value: `"2GB"` ++ Default value: `"2GiB"` + Minimum value: `0` -+ Unit: KB|MB|GB ++ Unit: KiB|MiB|GiB ### `compaction-pri` @@ -1532,14 +1612,14 @@ Configuration items related to `rocksdb.defaultcf`, `rocksdb.writecf`, and `rock ### `soft-pending-compaction-bytes-limit` + The soft limit on the pending compaction bytes. When `storage.flow-control.enable` is set to `true`, `storage.flow-control.soft-pending-compaction-bytes-limit` overrides this configuration item. -+ Default value: `"192GB"` -+ Unit: KB|MB|GB ++ Default value: `"192GiB"` ++ Unit: KiB|MiB|GiB ### `hard-pending-compaction-bytes-limit` + The hard limit on the pending compaction bytes. When `storage.flow-control.enable` is set to `true`, `storage.flow-control.hard-pending-compaction-bytes-limit` overrides this configuration item. -+ Default value: `"256GB"` -+ Unit: KB|MB|GB ++ Default value: `"256GiB"` ++ Unit: KiB|MiB|GiB ### `enable-compaction-guard` @@ -1550,14 +1630,14 @@ Configuration items related to `rocksdb.defaultcf`, `rocksdb.writecf`, and `rock ### `compaction-guard-min-output-file-size` + The minimum SST file size when the compaction guard is enabled. This configuration prevents SST files from being too small when the compaction guard is enabled. -+ Default value: `"8MB"` -+ Unit: KB|MB|GB ++ Default value: `"8MiB"` ++ Unit: KiB|MiB|GiB ### `compaction-guard-max-output-file-size` + The maximum SST file size when the compaction guard is enabled. The configuration prevents SST files from being too large when the compaction guard is enabled. This configuration overrides `target-file-size-base` for the same column family. -+ Default value: `"128MB"` -+ Unit: KB|MB|GB ++ Default value: `"128MiB"` ++ Unit: KiB|MiB|GiB ### `format-version` New in v6.2.0 @@ -1586,6 +1666,11 @@ Configuration items related to `rocksdb.defaultcf`, `rocksdb.writecf`, and `rock + Default value: `"0s"`, meaning that periodic compaction is disabled by default. + Unit: s(second)|h(hour)|d(day) +### `max-compactions` New in v6.6.0 + ++ The maximum number of concurrent compaction tasks. The value `0` means no limit. ++ Default value: `0` + ## rocksdb.defaultcf.titan Configuration items related to `rocksdb.defaultcf.titan`. @@ -1593,9 +1678,9 @@ Configuration items related to `rocksdb.defaultcf.titan`. ### `min-blob-size` + The smallest value stored in a Blob file. Values smaller than the specified size are stored in the LSM-Tree. -+ Default value: `"1KB"` ++ Default value: `"1KiB"` + Minimum value: `0` -+ Unit: KB|MB|GB ++ Unit: KiB|MiB|GiB ### `blob-file-compression` @@ -1610,23 +1695,23 @@ Configuration items related to `rocksdb.defaultcf.titan`. ### `blob-cache-size` + The cache size of a Blob file -+ Default value: `"0GB"` ++ Default value: `"0GiB"` + Minimum value: `0` -+ Unit: KB|MB|GB ++ Unit: KiB|MiB|GiB ### `min-gc-batch-size` + The minimum total size of Blob files required to perform GC for one time -+ Default value: `"16MB"` ++ Default value: `"16MiB"` + Minimum value: `0` -+ Unit: KB|MB|GB ++ Unit: KiB|MiB|GiB ### `max-gc-batch-size` + The maximum total size of Blob files allowed to perform GC for one time -+ Default value: `"64MB"` ++ Default value: `"64MiB"` + Minimum value: `0` -+ Unit: KB|MB|GB ++ Unit: KiB|MiB|GiB ### `discardable-ratio` @@ -1645,9 +1730,9 @@ Configuration items related to `rocksdb.defaultcf.titan`. ### `merge-small-file-threshold` + When the size of a Blob file is smaller than this value, the Blob file might still be selected for GC. In this situation, `discardable-ratio` is ignored. -+ Default value: `"8MB"` ++ Default value: `"8MiB"` + Minimum value: `0` -+ Unit: KB|MB|GB ++ Unit: KiB|MiB|GiB ### `blob-run-mode` @@ -1688,9 +1773,9 @@ Configuration items related to `raftdb` ### `max-manifest-file-size` + The maximum size of a RocksDB Manifest file -+ Default value: `"20MB"` ++ Default value: `"20MiB"` + Minimum value: `0` -+ Unit: B|KB|MB|GB ++ Unit: B|KiB|MiB|GiB ### `create-if-missing` @@ -1721,29 +1806,29 @@ Configuration items related to `raftdb` + The size limit of the archived WAL files. When the value is exceeded, the system deletes these files. + Default value: `0` + Minimum value: `0` -+ Unit: B|KB|MB|GB ++ Unit: B|KiB|MiB|GiB ### `max-total-wal-size` + The maximum RocksDB WAL size in total -+ Default value: `"4GB"` - + When `storage.engine="raft-kv"`, the default value is `"4GB"`. ++ Default value: + + When `storage.engine="raft-kv"`, the default value is `"4GiB"`. + When `storage.engine="partitioned-raft-kv"`, the default value is `1`. ### `compaction-readahead-size` + Controls whether to enable the readahead feature during RocksDB compaction and specify the size of readahead data. -+ If you use mechanical disks, it is recommended to set the value to `2MB` at least. ++ If you use mechanical disks, it is recommended to set the value to `2MiB` at least. + Default value: `0` + Minimum value: `0` -+ Unit: B|KB|MB|GB ++ Unit: B|KiB|MiB|GiB ### `writable-file-max-buffer-size` + The maximum buffer size used in WritableFileWrite -+ Default value: `"1MB"` ++ Default value: `"1MiB"` + Minimum value: `0` -+ Unit: B|KB|MB|GB ++ Unit: B|KiB|MiB|GiB ### `use-direct-io-for-flush-and-compaction` @@ -1763,31 +1848,43 @@ Configuration items related to `raftdb` ### `bytes-per-sync` + The rate at which OS incrementally synchronizes files to disk while these files are being written asynchronously -+ Default value: `"1MB"` ++ Default value: `"1MiB"` + Minimum value: `0` -+ Unit: B|KB|MB|GB ++ Unit: B|KiB|MiB|GiB ### `wal-bytes-per-sync` + The rate at which OS incrementally synchronizes WAL files to disk when the WAL files are being written -+ Default value: `"512KB"` ++ Default value: `"512KiB"` + Minimum value: `0` -+ Unit: B|KB|MB|GB ++ Unit: B|KiB|MiB|GiB ### `info-log-max-size` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated, and its function is replaced by the configuration item [`log.file.max-size`](#max-size-new-in-v540). + + The maximum size of Info logs -+ Default value: `"1GB"` ++ Default value: `"1GiB"` + Minimum value: `0` -+ Unit: B|KB|MB|GB ++ Unit: B|KiB|MiB|GiB ### `info-log-roll-time` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated. TiKV no longer supports automatic log splitting based on time. Instead, you can use the configuration item [`log.file.max-size`](#max-size-new-in-v540) to set the threshold for automatic log splitting based on file size. + + The interval at which Info logs are truncated. If the value is `0s`, logs are not truncated. + Default value: `"0s"` (which means logs are not truncated) ### `info-log-keep-log-file-num` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated, and its function is replaced by the configuration item [`log.file.max-backups`](#max-backups-new-in-v540). + + The maximum number of Info log files kept in RaftDB + Default value: `10` + Minimum value: `0` @@ -1799,6 +1896,10 @@ Configuration items related to `raftdb` ### `info-log-level` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated, and its function is replaced by the configuration item [`log.level`](#level-new-in-v540). + + Log levels of RaftDB + Default value: `"info"` @@ -1826,24 +1927,28 @@ Configuration items related to Raft Engine. ### `batch-compression-threshold` + Specifies the threshold size of a log batch. A log batch larger than this configuration is compressed. If you set this configuration item to `0`, compression is disabled. -+ Default value: `"8KB"` ++ Default value: `"8KiB"` ### `bytes-per-sync` +> **Warning:** +> +> Starting from v6.5.0, Raft Engine writes logs to disk directly without buffering. Therefore, this configuration item is deprecated and no longer functional. + + Specifies the maximum accumulative size of buffered writes. When this configuration value is exceeded, buffered writes are flushed to the disk. + If you set this configuration item to `0`, incremental sync is disabled. -+ Default value: `"4MB"` ++ Before v6.5.0, the default value is `"4MiB"`. ### `target-file-size` + Specifies the maximum size of log files. When a log file is larger than this value, it is rotated. -+ Default value: `"128MB"` ++ Default value: `"128MiB"` ### `purge-threshold` + Specifies the threshold size of the main log queue. When this configuration value is exceeded, the main log queue is purged. + This configuration can be used to adjust the disk space usage of Raft Engine. -+ Default value: `"10GB"` ++ Default value: `"10GiB"` ### `recovery-mode` @@ -1854,7 +1959,7 @@ Configuration items related to Raft Engine. ### `recovery-read-block-size` + The minimum I/O size for reading log files during recovery. -+ Default value: `"16KB"` ++ Default value: `"16KiB"` + Minimum value: `"512B"` ### `recovery-threads` @@ -1905,6 +2010,12 @@ Configuration items related to Raft Engine. + Determines whether to generate empty log files for log recycling in Raft Engine. When it is enabled, Raft Engine will automatically fill a batch of empty log files for log recycling during initialization, making log recycling effective immediately after initialization. + Default value: `false` +### `compression-level` New in v7.4.0 + ++ Sets the compression efficiency of the LZ4 algorithm used by Raft Engine when writing Raft log files. A lower value indicates faster compression speed but lower compression ratio. ++ Range: `[1, 16]` ++ Default value: `1` + ## security Configuration items related to security. @@ -2012,6 +2123,55 @@ Configuration items related to TiDB Lightning import and BR restore. + The garbage ratio threshold to trigger GC. + Default value: `1.1` +### `num-threads` New in v6.5.8, v7.1.4 and v7.5.1 + ++ The number of GC threads when `enable-compaction-filter` is `false`. ++ Default value: `1` + +## gc.auto-compaction + +Configures the behavior of TiKV automatic compaction. + +### `check-interval` New in v7.5.7 + ++ The interval at which TiKV checks whether to trigger automatic compaction. Within this interval, Regions that meet the automatic compaction conditions are processed based on priority. When the interval elapses, TiKV rescans Region information and recalculates priorities. ++ Default value: `"300s"` + +### `tombstone-num-threshold` New in v7.5.7 + ++ The number of RocksDB tombstones required to trigger TiKV automatic compaction. When the number of tombstones reaches this threshold, or when the percentage of tombstones reaches [`tombstone-percent-threshold`](#tombstone-percent-threshold-new-in-v757), TiKV triggers automatic compaction. ++ This configuration item takes effect only when [Compaction Filter](/garbage-collection-configuration.md) is disabled. ++ Default value: `10000` ++ Minimum value: `0` + +### `tombstone-percent-threshold` New in v7.5.7 + ++ The percentage of RocksDB tombstones required to trigger TiKV automatic compaction. When the percentage of tombstones reaches this threshold, or when the number of tombstones reaches [`tombstone-num-threshold`](#tombstone-num-threshold-new-in-v757), TiKV triggers automatic compaction. ++ This configuration item takes effect only when [Compaction Filter](/garbage-collection-configuration.md) is disabled. ++ Default value: `30` ++ Minimum value: `0` ++ Maximum value: `100` + +### `redundant-rows-threshold` New in v7.5.7 + ++ The number of redundant MVCC rows required to trigger TiKV automatic compaction. Redundant rows include RocksDB tombstones, TiKV stale versions, and TiKV deletion tombstones. When the number of redundant MVCC rows reaches this threshold, or when the percentage of these rows reaches [`redundant-rows-percent-threshold`](#redundant-rows-percent-threshold-new-in-v757), TiKV triggers automatic compaction. ++ This configuration item takes effect only when [Compaction Filter](/garbage-collection-configuration.md) is enabled. ++ Default value: `50000` ++ Minimum value: `0` + +### `redundant-rows-percent-threshold` New in v7.5.7 + ++ The percentage of redundant MVCC rows required to trigger TiKV automatic compaction. Redundant rows include RocksDB tombstones, TiKV stale versions, and TiKV deletion tombstones. When the number of redundant MVCC rows reaches [`redundant-rows-threshold`](#redundant-rows-threshold-new-in-v757), or when the percentage of these rows reaches `redundant-rows-percent-threshold`, TiKV triggers automatic compaction. ++ This configuration item takes effect only when [Compaction Filter](/garbage-collection-configuration.md) is enabled. ++ Default value: `20` ++ Minimum value: `0` ++ Maximum value: `100` + +### `bottommost-level-force` New in v7.5.7 + ++ Controls whether to force compaction on the bottommost files in RocksDB. ++ Default value: `true` + ## backup Configuration items related to BR backup. @@ -2032,7 +2192,7 @@ Configuration items related to BR backup. + The threshold of the backup SST file size. If the size of a backup file in a TiKV Region exceeds this threshold, the file is backed up to several files with the TiKV Region split into multiple Region ranges. Each of the files in the split Regions is the same size as `sst-max-size` (or slightly larger). + For example, when the size of a backup file in the Region of `[a,e)` is larger than `sst-max-size`, the file is backed up to several files with regions `[a,b)`, `[b,c)`, `[c,d)` and `[d,e)`, and the size of `[a,b)`, `[b,c)`, `[c,d)` is the same as that of `sst-max-size` (or slightly larger). -+ Default value: `"144MB"` ++ Default value: `"144MiB"` ### `enable-auto-tune` New in v5.4.0 @@ -2080,12 +2240,12 @@ Configuration items related to log backup. ### `initial-scan-pending-memory-quota` New in v6.2.0 + The quota of cache used for storing incremental scan data during log backup. -+ Default value: `min(Total machine memory * 10%, 512 MB)` ++ Default value: `min(Total machine memory * 10%, 512 MiB)` ### `initial-scan-rate-limit` New in v6.2.0 -+ The rate limit on throughput in an incremental data scan during log backup. -+ Default value: 60, indicating that the rate limit is 60 MB/s by default. ++ The rate limit on throughput in an incremental data scan during log backup, which means the maximum amount of data that can be read from the disk per second. Note that if you only specify a number (for example, `60`), the unit is Byte instead of KiB. ++ Default value: 60MiB ### `max-flush-interval` New in v6.2.0 @@ -2110,22 +2270,26 @@ Configuration items related to TiCDC. ### `min-ts-interval` + The interval at which Resolved TS is calculated and forwarded. -+ Default value: `"200ms"` ++ Default value: `"1s"`. + +> **Note:** +> +> In v6.5.0, the default value of `min-ts-interval` is changed from `"1s"` to `"200ms"` to reduce CDC latency. Starting from v6.5.1, this default value is changed back to `"1s"` to reduce network traffic. ### `old-value-cache-memory-quota` + The upper limit of memory usage by TiCDC old values. -+ Default value: `512MB` ++ Default value: `512MiB` ### `sink-memory-quota` + The upper limit of memory usage by TiCDC data change events. -+ Default value: `512MB` ++ Default value: `512MiB` ### `incremental-scan-speed-limit` + The maximum speed at which historical data is incrementally scanned. -+ Default value: `"128MB"`, which means 128 MB per second. ++ Default value: `"128MiB"`, which means 128 MiB per second. ### `incremental-scan-threads` @@ -2209,14 +2373,14 @@ Suppose that your machine on which TiKV is deployed has limited resources, for e #### `foreground-write-bandwidth` New in v6.0.0 + The soft limit on the bandwidth with which transactions write data. -+ Default value: `0KB` (which means no limit) -+ Recommended setting: Use the default value `0` in most cases unless the `foreground-cpu-time` setting is not enough to limit the write bandwidth. For such an exception, it is recommended to set the value smaller than `50MB` in the instance with 4 or less cores. ++ Default value: `0KiB` (which means no limit) ++ Recommended setting: Use the default value `0` in most cases unless the `foreground-cpu-time` setting is not enough to limit the write bandwidth. For such an exception, it is recommended to set the value smaller than `50MiB` in the instance with 4 or less cores. #### `foreground-read-bandwidth` New in v6.0.0 + The soft limit on the bandwidth with which transactions and the Coprocessor read data. -+ Default value: `0KB` (which means no limit) -+ Recommended setting: Use the default value `0` in most cases unless the `foreground-cpu-time` setting is not enough to limit the read bandwidth. For such an exception, it is recommended to set the value smaller than `20MB` in the instance with 4 or less cores. ++ Default value: `0KiB` (which means no limit) ++ Recommended setting: Use the default value `0` in most cases unless the `foreground-cpu-time` setting is not enough to limit the read bandwidth. For such an exception, it is recommended to set the value smaller than `20MiB` in the instance with 4 or less cores. ### Background Quota Limiter @@ -2237,21 +2401,13 @@ Suppose that your machine on which TiKV is deployed has limited resources, for e #### `background-write-bandwidth` New in v6.2.0 -> **Note:** -> -> This configuration item is returned in the result of `SHOW CONFIG`, but currently setting it does not take any effect. - + The soft limit on the bandwidth with which background transactions write data. -+ Default value: `0KB` (which means no limit) ++ Default value: `0KiB` (which means no limit) #### `background-read-bandwidth` New in v6.2.0 -> **Note:** -> -> This configuration item is returned in the result of `SHOW CONFIG`, but currently setting it does not take any effect. - + The soft limit on the bandwidth with which background transactions and the Coprocessor read data. -+ Default value: `0KB` (which means no limit) ++ Default value: `0KiB` (which means no limit) #### `enable-auto-tune` New in v6.2.0 @@ -2315,21 +2471,33 @@ Configuration items related to [Load Base Split](/configure-load-base-split.md). + Controls the traffic threshold at which a Region is identified as a hotspot. + Default value: - + `30MiB` per second when [`region-split-size`](#region-split-size) is less than 4 GB. - + `100MiB` per second when [`region-split-size`](#region-split-size) is greater than or equal to 4 GB. + + `30MiB` per second when [`region-split-size`](#region-split-size) is less than 4 GiB. + + `100MiB` per second when [`region-split-size`](#region-split-size) is greater than or equal to 4 GiB. ### `qps-threshold` + Controls the QPS threshold at which a Region is identified as a hotspot. + Default value: - + `3000` when [`region-split-size`](#region-split-size) is less than 4 GB. - + `7000` when [`region-split-size`](#region-split-size) is greater than or equal to 4 GB. + + `3000` when [`region-split-size`](#region-split-size) is less than 4 GiB. + + `7000` when [`region-split-size`](#region-split-size) is greater than or equal to 4 GiB. ### `region-cpu-overload-threshold-ratio` New in v6.2.0 + Controls the CPU usage threshold at which a Region is identified as a hotspot. + Default value: - + `0.25` when [`region-split-size`](#region-split-size) is less than 4 GB. - + `0.75` when [`region-split-size`](#region-split-size) is greater than or equal to 4 GB. + + `0.25` when [`region-split-size`](#region-split-size) is less than 4 GiB. + + `0.75` when [`region-split-size`](#region-split-size) is greater than or equal to 4 GiB. + +## memory New in v7.5.0 + +### `enable-heap-profiling` New in v7.5.0 + ++ Controls whether to enable Heap Profiling to track the memory usage of TiKV. ++ Default value: `true` + +### `profiling-sample-per-bytes` New in v7.5.0 + ++ Specifies the amount of data sampled by Heap Profiling each time, rounding up to the nearest power of 2. ++ Default value: `512KiB` diff --git a/tikv-control.md b/tikv-control.md index defb8e897b4ce..9162184cb8a19 100644 --- a/tikv-control.md +++ b/tikv-control.md @@ -1,7 +1,6 @@ --- title: TiKV Control User Guide summary: Use TiKV Control to manage a TiKV cluster. -aliases: ['/docs/dev/tikv-control/','/docs/dev/reference/tools/tikv-control/'] --- # TiKV Control User Guide @@ -641,9 +640,9 @@ it isn't easy to handle local data, start key:0101 overlap region: RegionInfo { region: id: 4 end_key: 7480000000000000FF0500000000000000F8 region_epoch { conf_ver: 1 version: 2 } peers { id: 5 store_id: 1 }, leader: Some(id: 5 store_id: 1) } -suggested operations: -tikv-ctl ldb --db=data/tikv-21107/db unsafe_remove_sst_file "data/tikv-21107/db/000014.sst" -tikv-ctl --db=data/tikv-21107/db tombstone -r 4 --pd +refer operations: +tikv-ctl ldb --db=/path/to/tikv/db unsafe_remove_sst_file 000014 +tikv-ctl --data-dir=/path/to/tikv tombstone -r 4 --pd -------------------------------------------------------- corruption analysis has completed ``` diff --git a/time-to-live.md b/time-to-live.md index 6754e999d19cd..b35670ddabacb 100644 --- a/time-to-live.md +++ b/time-to-live.md @@ -128,6 +128,14 @@ CREATE TABLE orders ( ## TTL job + + +> **Note:** +> +> In TiDB Serverless, the `TTL_JOB_INTERVAL` attribute for a table is fixed at `15m` and cannot be modified. This means that TiDB Serverless schedules a background job every 15 minutes to clean up expired data. + + + For each table with a TTL attribute, TiDB internally schedules a background job to clean up expired data. You can customize the execution period of these jobs by setting the `TTL_JOB_INTERVAL` attribute for the table. The following example sets the background cleanup jobs for the table `orders` to run once every 24 hours: ```sql @@ -159,7 +167,7 @@ The preceding statement allows TTL jobs to be scheduled only between 1:00 and 5: > **Note:** > -> This section is only applicable to TiDB Self-Hosted. Currently, TiDB Cloud does not provide TTL metrics. +> This section is only applicable to TiDB Self-Managed. Currently, TiDB Cloud does not provide TTL metrics.
    @@ -176,29 +184,32 @@ In addition, TiDB provides three tables to obtain more information about TTL job + The `mysql.tidb_ttl_table_status` table contains information about the previously executed TTL job and ongoing TTL job for all TTL tables ```sql - MySQL [(none)]> SELECT * FROM mysql.tidb_ttl_table_status LIMIT 1\G; + TABLE mysql.tidb_ttl_table_status LIMIT 1\G + ``` + + ``` *************************** 1. row *************************** table_id: 85 - parent_table_id: 85 + parent_table_id: 85 table_statistics: NULL - last_job_id: 0b4a6d50-3041-4664-9516-5525ee6d9f90 - last_job_start_time: 2023-02-15 20:43:46 + last_job_id: 0b4a6d50-3041-4664-9516-5525ee6d9f90 + last_job_start_time: 2023-02-15 20:43:46 last_job_finish_time: 2023-02-15 20:44:46 - last_job_ttl_expire: 2023-02-15 19:43:46 + last_job_ttl_expire: 2023-02-15 19:43:46 last_job_summary: {"total_rows":4369519,"success_rows":4369519,"error_rows":0,"total_scan_task":64,"scheduled_scan_task":64,"finished_scan_task":64} current_job_id: NULL current_job_owner_id: NULL current_job_owner_addr: NULL - current_job_owner_hb_time: NULL + current_job_owner_hb_time: NULL current_job_start_time: NULL current_job_ttl_expire: NULL - current_job_state: NULL + current_job_state: NULL current_job_status: NULL current_job_status_update_time: NULL 1 row in set (0.040 sec) ``` - The column `table_id` is the ID of the partitioned table, and the `parent_table_id` is the ID of the table, corresponding with the ID in `infomation_schema.tables`. If the table is not a partitioned table, the two IDs are the same. + The column `table_id` is the ID of the partitioned table, and the `parent_table_id` is the ID of the table, corresponding with the ID in [`information_schema.tables`](/information-schema/information-schema-tables.md). If the table is not a partitioned table, the two IDs are the same. The columns `{last, current}_job_{start_time, finish_time, ttl_expire}` describe respectively the start time, finish time, and expiration time used by the TTL job of the last or current execution. The `last_job_summary` column describes the execution status of the last TTL task, including the total number of rows, the number of successful rows, and the number of failed rows. @@ -206,25 +217,28 @@ In addition, TiDB provides three tables to obtain more information about TTL job + The `mysql.tidb_ttl_job_history` table contains information about the TTL jobs that have been executed. The record of TTL job history is kept for 90 days. ```sql - MySQL [(none)]> SELECT * FROM mysql.tidb_ttl_job_history LIMIT 1\G; + TABLE mysql.tidb_ttl_job_history LIMIT 1\G + ``` + + ``` *************************** 1. row *************************** - job_id: f221620c-ab84-4a28-9d24-b47ca2b5a301 - table_id: 85 + job_id: f221620c-ab84-4a28-9d24-b47ca2b5a301 + table_id: 85 parent_table_id: 85 - table_schema: test_schema - table_name: TestTable - partition_name: NULL + table_schema: test_schema + table_name: TestTable + partition_name: NULL create_time: 2023-02-15 17:43:46 finish_time: 2023-02-15 17:45:46 - ttl_expire: 2023-02-15 16:43:46 - summary_text: {"total_rows":9588419,"success_rows":9588419,"error_rows":0,"total_scan_task":63,"scheduled_scan_task":63,"finished_scan_task":63} - expired_rows: 9588419 - deleted_rows: 9588419 + ttl_expire: 2023-02-15 16:43:46 + summary_text: {"total_rows":9588419,"success_rows":9588419,"error_rows":0,"total_scan_task":63,"scheduled_scan_task":63,"finished_scan_task":63} + expired_rows: 9588419 + deleted_rows: 9588419 error_delete_rows: 0 - status: finished + status: finished ``` - The column `table_id` is the ID of the partitioned table, and the `parent_table_id` is the ID of the table, corresponding with the ID in `infomation_schema.tables`. `table_schema`, `table_name`, and `partition_name` correspond to the database, table name, and partition name. `create_time`, `finish_time`, and `ttl_expire` indicate the creation time, end time, and expiration time of the TTL task. `expired_rows` and `deleted_rows` indicate the number of expired rows and the number of rows deleted successfully. + The column `table_id` is the ID of the partitioned table, and the `parent_table_id` is the ID of the table, corresponding with the ID in `information_schema.tables`. `table_schema`, `table_name`, and `partition_name` correspond to the database, table name, and partition name. `create_time`, `finish_time`, and `ttl_expire` indicate the creation time, end time, and expiration time of the TTL task. `expired_rows` and `deleted_rows` indicate the number of expired rows and the number of rows deleted successfully. ## Compatibility with TiDB tools @@ -242,7 +256,7 @@ TTL can be used with other TiDB migration, backup, and recovery tools. | :-- | :---- | | [`FLASHBACK TABLE`](/sql-statements/sql-statement-flashback-table.md) | `FLASHBACK TABLE` will set the `TTL_ENABLE` attribute of the tables to `OFF`. This prevents TiDB from immediately deleting expired data after the flashback. You need to manually turn on the `TTL_ENABLE` attribute to re-enable TTL for each table. | | [`FLASHBACK DATABASE`](/sql-statements/sql-statement-flashback-database.md) | `FLASHBACK DATABASE` will set the `TTL_ENABLE` attribute of the tables to `OFF`, and the `TTL_ENABLE` attribute will not be modified. This prevents TiDB from immediately deleting expired data after the flashback. You need to manually turn on the `TTL_ENABLE` attribute to re-enable TTL for each table. | -| [`FLASHBACK CLUSTER TO TIMESTAMP`](/sql-statements/sql-statement-flashback-to-timestamp.md) | `FLASHBACK CLUSTER TO TIMESTAMP` will set the system variable [`TIDB_TTL_JOB_ENABLE`](/system-variables.md#tidb_ttl_job_enable-new-in-v650) to `OFF` and do not change the value of the `TTL_ENABLE` attribute. | +| [`FLASHBACK CLUSTER`](/sql-statements/sql-statement-flashback-cluster.md) | `FLASHBACK CLUSTER` will set the system variable [`TIDB_TTL_JOB_ENABLE`](/system-variables.md#tidb_ttl_job_enable-new-in-v650) to `OFF` and do not change the value of the `TTL_ENABLE` attribute. | ## Limitations @@ -252,7 +266,6 @@ Currently, the TTL feature has the following limitations: * A table with the TTL attribute does not support being referenced by other tables as the primary table in a foreign key constraint. * It is not guaranteed that all expired data is deleted immediately. The time when expired data is deleted depends on the scheduling interval and scheduling window of the background cleanup job. * For tables that use [clustered indexes](/clustered-indexes.md), if the primary key is neither an integer nor a binary string type, the TTL job cannot be split into multiple tasks. This will cause the TTL job to be executed sequentially on a single TiDB node. If the table contains a large amount of data, the execution of the TTL job might become slow. -* TTL is not available for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). ## FAQs diff --git a/tispark-deployment-topology.md b/tispark-deployment-topology.md index 445ec4b744d9d..cc79d3f4087ff 100644 --- a/tispark-deployment-topology.md +++ b/tispark-deployment-topology.md @@ -25,6 +25,10 @@ For more information about the TiSpark architecture and how to use it, see [TiSp | TiSpark | 3 | 8 VCore 16GB * 1 | 10.0.1.21 (master)
    10.0.1.22 (worker)
    10.0.1.23 (worker) | Default port
    Global directory configuration | | Monitoring & Grafana | 1 | 4 VCore 8GB * 1 500GB (ssd) | 10.0.1.11 | Default port
    Global directory configuration | +> **Note:** +> +> The IP addresses of the instances are given as examples only. In your actual deployment, replace the IP addresses with your actual IP addresses. + ## Topology templates - [Simple TiSpark topology template](https://github.com/pingcap/docs/blob/master/config-templates/simple-tispark.yaml) diff --git a/tispark-overview.md b/tispark-overview.md index 8323afd2c3740..2f79d6642ca1e 100644 --- a/tispark-overview.md +++ b/tispark-overview.md @@ -1,20 +1,14 @@ --- title: TiSpark User Guide summary: Use TiSpark to provide an HTAP solution to serve as a one-stop solution for both online transactions and analysis. -aliases: ['/docs/dev/tispark-overview/','/docs/dev/reference/tispark/','/docs/dev/get-started-with-tispark/','/docs/dev/how-to/get-started/tispark/','/docs/dev/how-to/deploy/tispark/','/tidb/dev/get-started-with-tispark/','/tidb/stable/get-started-with-tispark'] --- # TiSpark User Guide -![TiSpark architecture](/media/tispark-architecture.png) - -## TiSpark vs TiFlash - -[TiSpark](https://github.com/pingcap/tispark) is a thin layer built for running Apache Spark on top of TiDB/TiKV to answer the complex OLAP queries. It takes advantages of both the Spark platform and the distributed TiKV cluster and seamlessly glues to TiDB, the distributed OLTP database, to provide a Hybrid Transactional/Analytical Processing (HTAP) solution to serve as a one-stop solution for both online transactions and analysis. - -[TiFlash](/tiflash/tiflash-overview.md) is another tool that enables HTAP. Both TiFlash and TiSpark allow the use of multiple hosts to execute OLAP queries on OLTP data. TiFlash stores data in a columnar format, which allows more efficient analytical queries. TiFlash and TiSpark can be used together. - -## What is TiSpark +> **Warning:** +> +> - TiSpark does not guarantee compatibility with TiDB v7.0.0 and later versions. +> - TiSpark does not guarantee compatibility with Spark v3.4.0 and later versions. TiSpark depends on the TiKV cluster and the PD cluster. You also need to set up a Spark cluster. This document provides a brief introduction to how to setup and use TiSpark. It requires some basic knowledge of Apache Spark. For more information, see [Apache Spark website](https://spark.apache.org/docs/latest/index.html). @@ -34,6 +28,16 @@ Also, TiSpark supports distributed writes to TiKV. Compared with writes to TiDB > > Because TiSpark accesses TiKV directly, the access control mechanisms used by TiDB Server are not applicable to TiSpark. Since TiSpark v2.5.0, TiSpark supports user authentication and authorization, for more information, see [Security](/tispark-overview.md#security). +The following diagram shows the architecture of TiSpark. + +![TiSpark architecture](/media/tispark-architecture.png) + +## TiSpark vs TiFlash + +[TiSpark](https://github.com/pingcap/tispark) is a thin layer built for running Apache Spark on top of TiDB/TiKV to answer complex OLAP queries. It takes advantage of both the Spark platform and the distributed TiKV cluster and seamlessly integrates with TiDB, the distributed OLTP database, to provide a Hybrid Transactional/Analytical Processing (HTAP) solution to serve as a one-stop solution for both online transactions and analysis. + +[TiFlash](/tiflash/tiflash-overview.md) is another tool that enables HTAP. Both TiFlash and TiSpark allow the use of multiple hosts to execute OLAP queries on OLTP data. TiFlash stores data in a columnar format, which allows more efficient analytical queries. TiFlash and TiSpark can be used together. + ## Requirements + TiSpark supports Spark >= 2.3. @@ -98,8 +102,9 @@ You can choose TiSpark version according to your TiDB and Spark version. | 2.5.x | 5.x, 4.x | 3.0.x, 3.1.x | 2.12 | | 3.0.x | 5.x, 4.x | 3.0.x, 3.1.x, 3.2.x|2.12| | 3.1.x | 6.x, 5.x, 4.x | 3.0.x, 3.1.x, 3.2.x, 3.3.x|2.12| +| 3.2.x | 6.x, 5.x, 4.x | 3.0.x, 3.1.x, 3.2.x, 3.3.x|2.12| -TiSpark 2.4.4, 2.5.2, 3.0.2 and 3.1.1 are the latest stable versions and are highly recommended. +TiSpark 2.4.4, 2.5.2, 3.0.2, 3.1.1, and 3.2.3 are the latest stable versions and are highly recommended. ### Get TiSpark jar @@ -130,11 +135,11 @@ mvn clean install -Dmaven.test.skip=true -Pspark3.2.1 The Artifact ID of TiSpark varies with TiSpark versions. -| TiSpark version | Artifact ID | -|-------------------------------| -------------------------------------------------- | +| TiSpark version | Artifact ID | +|--------------------------------| -------------------------------------------------- | | 2.4.x-\${scala_version}, 2.5.0 | tispark-assembly | -| 2.5.1 | tispark-assembly-\${spark_version} | -| 3.0.x, 3.1.x | tispark-assembly-\${spark_version}-\${scala_version} | +| 2.5.1 | tispark-assembly-\${spark_version} | +| 3.0.x, 3.1.x, 3.2.x | tispark-assembly-\${spark_version}-\${scala_version} | ## Getting started @@ -148,9 +153,9 @@ Add the following configuration in `spark-defaults.conf`: ``` spark.sql.extensions org.apache.spark.sql.TiExtensions -spark.tispark.pd.addresses ${your_pd_adress} +spark.tispark.pd.addresses ${your_pd_address} spark.sql.catalog.tidb_catalog org.apache.spark.sql.catalyst.catalog.TiCatalog -spark.sql.catalog.tidb_catalog.pd.addresses ${your_pd_adress} +spark.sql.catalog.tidb_catalog.pd.addresses ${your_pd_address} ``` Start spark-shell with the `--jars` option. @@ -201,7 +206,7 @@ customerDF.write See [Data Source API User Guide](https://github.com/pingcap/tispark/blob/master/docs/features/datasource_api_userguide.md) for more details. -You can also write with Spark SQL since TiSpark 3.1. See [insert SQL](https://github.com/pingcap/tispark/blob/master/docs/features/insert_sql_userguide.md) for more details. +Starting from TiSpark 3.1, you can write data to TiKV using Spark SQL. For more information, see [insert SQL](https://github.com/pingcap/tispark/blob/master/docs/features/insert_sql_userguide.md). ### Write data using JDBC DataSource diff --git a/tiup/customized-montior-in-tiup-environment.md b/tiup/customized-montior-in-tiup-environment.md index c2f90850f9412..57e5573a03f02 100644 --- a/tiup/customized-montior-in-tiup-environment.md +++ b/tiup/customized-montior-in-tiup-environment.md @@ -70,7 +70,7 @@ After the preceding configuration is done, when you deploy, scale out, scale in, action: drop ``` -After the preceding configuration is done, when you deploy, scale out, scale in, or reload a TiDB cluster, TiUP adds the `additional_scrape_conf` field to the corresponding parameters of the Prometheus configuration file. +After the preceding configuration is done, when you deploy, scale out, scale in, or reload a TiDB cluster, TiUP adds the content of the `additional_scrape_conf` field to the corresponding parameters of the Prometheus configuration file. ## Customize Grafana configurations @@ -116,7 +116,7 @@ After the preceding configuration is done, when you deploy, scale out, scale in, smtp.skip_verify: true ``` -After the preceding configuration is done, when you deploy, scale out, scale in, or reload a TiDB cluster, TiUP adds the `config` field to the Grafana configuration file `grafana.ini`. +After the preceding configuration is done, when you deploy, scale out, scale in, or reload a TiDB cluster, TiUP adds the content of the `config` field to the Grafana configuration file `grafana.ini`. ## Customize Alertmanager configurations @@ -135,4 +135,4 @@ alertmanager_servers: ssh_port: 22 ``` -After the preceding configuration is done, when you deploy, scale out, scale in, or reload a TiDB cluster, TiUP adds the `listen_host` field to `--web.listen-address` in Alertmanager startup parameters. +After the preceding configuration is done, when you deploy, scale out, scale in, or reload a TiDB cluster, TiUP adds the content of the `listen_host` field to `--web.listen-address` in Alertmanager startup parameters. diff --git a/tiup/tiup-bench.md b/tiup/tiup-bench.md index c69b483af73eb..051f3d835f1a6 100644 --- a/tiup/tiup-bench.md +++ b/tiup/tiup-bench.md @@ -1,7 +1,6 @@ --- title: Stress Test TiDB Using TiUP Bench Component summary: Learn how to stress test TiDB with TPC-C, TPC-H, CH, RawSQL, and YCSB workloads using TiUP. -aliases: ['/docs/dev/tiup/tiup-bench/','/docs/dev/reference/tools/tiup/bench/'] --- # Stress Test TiDB Using TiUP Bench Component diff --git a/tiup/tiup-cluster-topology-reference.md b/tiup/tiup-cluster-topology-reference.md index dd9a493c8da02..ebf3f8868b03d 100644 --- a/tiup/tiup-cluster-topology-reference.md +++ b/tiup/tiup-cluster-topology-reference.md @@ -1,5 +1,6 @@ --- title: Topology Configuration File for TiDB Deployment Using TiUP +summary: TiUP uses a topology file to deploy or modify the cluster topology for TiDB. It also deploys monitoring servers like Prometheus, Grafana, and Alertmanager. The topology file contains sections for global configuration, monitoring services, component versions, and more. Each section specifies the machines to which the corresponding services are deployed and their configurations. --- # Topology Configuration File for TiDB Deployment Using TiUP @@ -8,6 +9,8 @@ To deploy or scale TiDB using TiUP, you need to provide a topology file ([sample Similarly, to modify the cluster topology, you need to modify the topology file. The difference is that, after the cluster is deployed, you can only modify a part of the fields in the topology file. This document introduces each section of the topology file and each field in each section. +When you deploy a TiDB cluster using TiUP, TiUP also deploys monitoring servers, such as Prometheus, Grafana, and Alertmanager. In the meantime, if you scale out this cluster, TiUP also adds the new nodes into monitoring scope. To customize the configurations of the preceding monitoring servers, you can follow the instructions in [Customize Configurations of Monitoring Servers](/tiup/customized-montior-in-tiup-environment.md). + ## File structure A topology configuration file for TiDB deployment using TiUP might contain the following sections: @@ -15,6 +18,7 @@ A topology configuration file for TiDB deployment using TiUP might contain the f - [global](#global): The cluster's global configuration. Some of the configuration items use the default values and you can configure them separately in each instance. - [monitored](#monitored): Configuration for monitoring services, namely, the blackbox_exporter and the `node_exporter`. On each machine, a `node_exporter` and a `blackbox_exporter` are deployed. - [server_configs](#server_configs): Components' global configuration. You can configure each component separately. If an instance has a configuration item with the same name, the instance's configuration item will take effect. +- [component_versions](#component_versions): Component version. You can configure it when a component does not use the cluster version. This section is introduced in tiup-cluster v1.14.0. - [pd_servers](#pd_servers): The configuration of the PD instance. This configuration specifies the machines to which the PD component is deployed. - [tidb_servers](#tidb_servers): The configuration of the TiDB instance. This configuration specifies the machines to which the TiDB component is deployed. - [tikv_servers](#tikv_servers): The configuration of the TiKV instance. This configuration specifies the machines to which the TiKV component is deployed. @@ -40,6 +44,8 @@ The `global` section corresponds to the cluster's global configuration and has t - `enable_tls`: Specifies whether to enable TLS for the cluster. After TLS is enabled, the generated TLS certificate must be used for connections between components or between the client and the component. The default value is `false`. +- `listen_host`: Specifies the default listening IP address. If it is empty, each instance automatically sets it to `::` or `0.0.0.0` based on whether its `host` field contains `:`. This field is introduced in tiup-cluster v1.14.0. + - `deploy_dir`: The deployment directory of each component. The default value is `"deployed"`. Its application rules are as follows: - If the absolute path of `deploy_dir` is configured at the instance level, the actual deployment directory is `deploy_dir` configured for the instance. @@ -153,6 +159,43 @@ server_configs: The above configuration specifies the global configuration of TiDB and TiKV. +### `component_versions` + +> **Note:** +> +> For components that share a version number, such as TiDB, TiKV, PD, and TiCDC, there are no complete tests to ensure that they work properly in a mixed-version deployment scenario. Ensure that you use this section only in test environments, or with the help of [technical support](/support.md). + +`component_versions` is used to specify the version number of a certain component. + +- When `component_versions` is not configured, each component either uses the same version number as the TiDB cluster (such as PD and TiKV), or uses the latest version (such as Alertmanager). +- When `component_versions` is configured, the corresponding component will use the specified version, and this version will be used in subsequent cluster scaling and upgrade operations. + +Make sure you only configure it when you need to use a specific version of a component. + +`component_versions` contains the following fields: + +- `tikv`: The version of the TiKV component +- `tiflash`: The version of the TiFlash component +- `pd`: The version of the PD component +- `tidb_dashboard`: The version of the standalone TiDB Dashboard component +- `pump`: The version of the Pump component +- `drainer`: The version of the Drainer component +- `cdc`: The version of the CDC component +- `kvcdc`: The version of the TiKV-CDC component +- `tiproxy`: The version of the TiProxy component +- `prometheus`: The version of the Prometheus component +- `grafana`: The version of the Grafana component +- `alertmanager`: The version of the Alertmanager component + +The following is an example configuration for `component_versions`: + +```yaml +component_versions: + kvcdc: "v1.1.1" +``` + +The preceding configuration specifies the version number of TiKV-CDC to be `v1.1.1`. + ### `pd_servers` `pd_servers` specifies the machines to which PD services are deployed. It also specifies the service configuration on each machine. `pd_servers` is an array, and each element of the array contains the following fields: @@ -323,7 +366,7 @@ tikv_servers: - `ssh_port`: Specifies the SSH port to connect to the target machine for operations. If it is not specified, the `ssh_port` of the `global` section is used. -- `tcp_port`: The port of the TiFlash TCP service. The default value is `9000`. +- `tcp_port`: The port of the TiFlash TCP service for internal testing purposes. The default value is `9000`. Starting from TiUP v1.12.5, this configuration item does not take effect on clusters that are v7.1.0 or later. - `flash_service_port`: The port via which TiFlash provides services. TiDB reads data from TiFlash via this port. The default value is `3930`. @@ -357,7 +400,6 @@ After the deployment, for the fields above, you can only add directories to `dat - `host` - `tcp_port` -- `http_port` - `flash_service_port` - `flash_proxy_port` - `flash_proxy_status_port` @@ -667,6 +709,10 @@ tispark_workers: - `resource_control`: Resource control for the service. If this field is configured, the field content is merged with the `resource_control` content in `global` (if the two fields overlap, the content of this field takes effect). Then, a systemd configuration file is generated and sent to the machine specified in `host`. The configuration rules of `resource_control` are the same as the `resource_control` content in `global`. +- `additional_args`: Introduced in TiUP v1.15.0, this field configures additional parameters for running Prometheus. This field is an array, and each element of the array is a Prometheus running parameter. For example, to enable the Prometheus hot reload feature, you can set this field to `--web.enable-lifecycle`. + +- `additional_scrape_conf`: Customized Prometheus scrape configuration. When you deploy, scale out, scale in, or reload a TiDB cluster, TiUP adds the content of the `additional_scrape_conf` field to the corresponding parameters of the Prometheus configuration file. For more information, see [Customize Prometheus scrape configuration](/tiup/customized-montior-in-tiup-environment.md#customize-prometheus-scrape-configuration). + For the above fields, you cannot modify these configured fields after the deployment: - `host` @@ -683,6 +729,8 @@ A `monitoring_servers` configuration example is as follows: monitoring_servers: - host: 10.0.1.11 rule_dir: /local/rule/dir + additional_args: + - --web.enable-lifecycle remote_config: remote_write: - queue_config: @@ -724,6 +772,8 @@ monitoring_servers: - `resource_control`: Resource control for the service. If this field is configured, the field content is merged with the `resource_control` content in `global` (if the two fields overlap, the content of this field takes effect). Then, a systemd configuration file is generated and sent to the machine specified in `host`. The configuration rules of `resource_control` are the same as the `resource_control` content in `global`. +- `config`: This field is used to add custom configurations to Grafana. When you deploy, scale out, scale in, or reload a TiDB cluster, TiUP adds the content of the `config` field to the Grafana configuration file `grafana.ini`. For more information, see [Customize other Grafana configurations](/tiup/customized-montior-in-tiup-environment.md#customize-other-grafana-configurations). + > **Note:** > > If the `dashboard_dir` field of `grafana_servers` is configured, after executing the `tiup cluster rename` command to rename the cluster, you need to perform the following operations: @@ -775,6 +825,8 @@ grafana_servers: - `resource_control`: Resource control for the service. If this field is configured, the field content is merged with the `resource_control` content in `global` (if the two fields overlap, the content of this field takes effect). Then, a systemd configuration file is generated and sent to the machine specified in `host`. The configuration rules of `resource_control` are the same as the `resource_control` content in `global`. +- `listen_host`: Specifies the listening address so that Alertmanager can be accessed through a proxy. It is recommended to configure it as `0.0.0.0`. For more information, see [Customize Alertmanager configurations](/tiup/customized-montior-in-tiup-environment.md#customize-alertmanager-configurations). + For the above fields, you cannot modify these configured fields after the deployment: - `host` diff --git a/tiup/tiup-cluster.md b/tiup/tiup-cluster.md index 3d70171b5eabf..c92b632827e25 100644 --- a/tiup/tiup-cluster.md +++ b/tiup/tiup-cluster.md @@ -1,7 +1,6 @@ --- title: Deploy and Maintain an Online TiDB Cluster Using TiUP summary: Learns how to deploy and maintain an online TiDB cluster using TiUP. -aliases: ['/docs/dev/tiup/tiup-cluster/','/docs/dev/reference/tools/tiup/cluster/'] --- # Deploy and Maintain an Online TiDB Cluster Using TiUP @@ -62,7 +61,7 @@ To deploy the cluster, run the `tiup cluster deploy` command. The usage of the c tiup cluster deploy [flags] ``` -This command requires you to provide the cluster name, the TiDB cluster version (such as `v7.4.0`), and a topology file of the cluster. +This command requires you to provide the cluster name, the TiDB cluster version (such as `v{{{ .tidb-version }}}`), and a topology file of the cluster. To write a topology file, refer to [the example](https://github.com/pingcap/tiup/blob/master/embed/examples/cluster/topology.example.yaml). The following file is an example of the simplest topology: @@ -119,12 +118,12 @@ tidb_servers: ... ``` -Save the file as `/tmp/topology.yaml`. If you want to use TiDB v7.4.0 and your cluster name is `prod-cluster`, run the following command: +Save the file as `/tmp/topology.yaml`. If you want to use TiDB v{{{ .tidb-version }}} and your cluster name is `prod-cluster`, run the following command: {{< copyable "shell-regular" >}} ```shell -tiup cluster deploy -p prod-cluster v7.4.0 /tmp/topology.yaml +tiup cluster deploy -p prod-cluster v{{{ .tidb-version }}} /tmp/topology.yaml ``` During the execution, TiUP asks you to confirm your topology again and requires the root password of the target machine (the `-p` flag means inputting password): @@ -132,7 +131,7 @@ During the execution, TiUP asks you to confirm your topology again and requires ```bash Please confirm your topology: TiDB Cluster: prod-cluster -TiDB Version: v7.4.0 +TiDB Version: v{{{ .tidb-version }}} Type Host Ports OS/Arch Directories ---- ---- ----- ------- ----------- pd 172.16.5.134 2379/2380 linux/x86_64 deploy/pd-2379,data/pd-2379 @@ -175,7 +174,7 @@ tiup cluster list Starting /root/.tiup/components/cluster/v1.12.3/cluster list Name User Version Path PrivateKey ---- ---- ------- ---- ---------- -prod-cluster tidb v7.4.0 /root/.tiup/storage/cluster/clusters/prod-cluster /root/.tiup/storage/cluster/clusters/prod-cluster/ssh/id_rsa +prod-cluster tidb v{{{ .tidb-version }}} /root/.tiup/storage/cluster/clusters/prod-cluster /root/.tiup/storage/cluster/clusters/prod-cluster/ssh/id_rsa ``` ## Start the cluster @@ -205,7 +204,7 @@ tiup cluster display prod-cluster ``` Starting /root/.tiup/components/cluster/v1.12.3/cluster display prod-cluster TiDB Cluster: prod-cluster -TiDB Version: v7.4.0 +TiDB Version: v{{{ .tidb-version }}} ID Role Host Ports OS/Arch Status Data Dir Deploy Dir -- ---- ---- ----- ------- ------ -------- ---------- 172.16.5.134:3000 grafana 172.16.5.134 3000 linux/x86_64 Up - deploy/grafana-3000 @@ -279,7 +278,7 @@ tiup cluster display prod-cluster ``` Starting /root/.tiup/components/cluster/v1.12.3/cluster display prod-cluster TiDB Cluster: prod-cluster -TiDB Version: v7.4.0 +TiDB Version: v{{{ .tidb-version }}} ID Role Host Ports OS/Arch Status Data Dir Deploy Dir -- ---- ---- ----- ------- ------ -------- ---------- 172.16.5.134:3000 grafana 172.16.5.134 3000 linux/x86_64 Up - deploy/grafana-3000 @@ -390,12 +389,12 @@ Global Flags: -y, --yes Skip all confirmations and assumes 'yes' ``` -For example, the following command upgrades the cluster to v7.4.0: +For example, the following command upgrades the cluster to v{{{ .tidb-version }}}: {{< copyable "shell-regular" >}} ```bash -tiup cluster upgrade tidb-test v7.4.0 +tiup cluster upgrade tidb-test v{{{ .tidb-version }}} ``` ## Update configuration @@ -580,11 +579,11 @@ tiup cluster audit Starting component `cluster`: /home/tidb/.tiup/components/cluster/v1.12.3/cluster audit ID Time Command -- ---- ------- -4BLhr0 2023-10-12T23:55:09+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v7.4.0 /tmp/topology.yaml -4BKWjF 2023-10-12T23:36:57+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v7.4.0 /tmp/topology.yaml -4BKVwH 2023-10-12T23:02:08+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v7.4.0 /tmp/topology.yaml -4BKKH1 2023-10-12T16:39:04+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster destroy test -4BKKDx 2023-10-12T16:36:57+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v7.4.0 /tmp/topology.yaml +4BLhr0 {{{ .tidb-release-date }}}T23:55:09+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v{{{ .tidb-version }}} /tmp/topology.yaml +4BKWjF {{{ .tidb-release-date }}}T23:36:57+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v{{{ .tidb-version }}} /tmp/topology.yaml +4BKVwH {{{ .tidb-release-date }}}T23:02:08+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v{{{ .tidb-version }}} /tmp/topology.yaml +4BKKH1 {{{ .tidb-release-date }}}T16:39:04+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster destroy test +4BKKDx {{{ .tidb-release-date }}}T16:36:57+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v{{{ .tidb-version }}} /tmp/topology.yaml ``` The first column is `audit-id`. To view the execution log of a certain command, pass the `audit-id` of a command as the flag as follows: @@ -700,7 +699,7 @@ All operations above performed on the cluster machine use the SSH client embedde Then you can use the `--ssh=system` command-line flag to enable the system-native command-line tool: -- Deploy a cluster: `tiup cluster deploy --ssh=system`. Fill in the name of your cluster for ``, the TiDB version to be deployed (such as `v7.4.0`) for ``, and the topology file for ``. +- Deploy a cluster: `tiup cluster deploy --ssh=system`. Fill in the name of your cluster for ``, the TiDB version to be deployed (such as `v{{{ .tidb-version }}}`) for ``, and the topology file for ``. - Start a cluster: `tiup cluster start --ssh=system` - Upgrade a cluster: `tiup cluster upgrade ... --ssh=system` diff --git a/tiup/tiup-command-clean.md b/tiup/tiup-command-clean.md index c1495e26fa7b0..6de01bf550ee3 100644 --- a/tiup/tiup-command-clean.md +++ b/tiup/tiup-command-clean.md @@ -1,5 +1,6 @@ --- title: tiup clean +summary: The "tiup clean" command clears data generated during component operation. The syntax is "tiup clean [name] [flags]", with the option to use "--all" to clear all operation records. --- # tiup clean diff --git a/tiup/tiup-command-completion.md b/tiup/tiup-command-completion.md index ba14ca86faef6..58e1b06f58111 100644 --- a/tiup/tiup-command-completion.md +++ b/tiup/tiup-command-completion.md @@ -1,5 +1,6 @@ --- title: tiup completion +summary: TiUP provides the `tiup completion` command to generate a configuration file for automatic command-line completion, supporting `bash` and `zsh` commands. To complete `bash` commands, install `bash-completion` and use the `tiup completion ` syntax to set the shell type. For `bash`, write the command into a file and source it in `.bash_profile`. For `zsh`, use the `tiup completion zsh` command. --- # tiup completion diff --git a/tiup/tiup-command-env.md b/tiup/tiup-command-env.md index 0219eb878f0c2..67c8ca2222e4a 100644 --- a/tiup/tiup-command-env.md +++ b/tiup/tiup-command-env.md @@ -1,5 +1,6 @@ --- title: tiup env +summary: TiUP provides flexible and customized interfaces using environment variables. The `tiup env` command queries user-defined environment variables and their values. Use `tiup env [name1...N]` to view specified variables, or all by default. No options. Output is a list of "{key}"="{value}" if not specified, or the "{value}" list in order if specified. Empty value means TiUP uses default. --- # tiup env diff --git a/tiup/tiup-command-help.md b/tiup/tiup-command-help.md index f398f9e120ddb..6ef3033fafa33 100644 --- a/tiup/tiup-command-help.md +++ b/tiup/tiup-command-help.md @@ -1,5 +1,6 @@ --- title: tiup help +summary: The TiUP command-line interface provides a wealth of help information, which can be accessed using the `help` command or the `--help` option. By using the `tiup help [command]` syntax, users can specify which command's help information they want to view. If no command is specified, the help information for TiUP is displayed. There are no options for this command, and the output will be the help information for the specified command or for TiUP. --- # tiup help diff --git a/tiup/tiup-command-install.md b/tiup/tiup-command-install.md index 5244441c5451e..7490408968b9c 100644 --- a/tiup/tiup-command-install.md +++ b/tiup/tiup-command-install.md @@ -1,5 +1,6 @@ --- title: tiup install +summary: The tiup install command is used to download and decompress component packages from the mirror repository for later use. If the component does not exist in the repository, it tries to download it and then runs it automatically. The syntax is "tiup install [:version] [component2...N] [flags]". There are no options, and the output includes download information or error messages if the component or version does not exist. --- # tiup install diff --git a/tiup/tiup-command-list.md b/tiup/tiup-command-list.md index 9f9a0b54ed8dc..c419a41bf5762 100644 --- a/tiup/tiup-command-list.md +++ b/tiup/tiup-command-list.md @@ -1,5 +1,6 @@ --- title: tiup list +summary: The `tiup list` command is used to get the list of available components of a mirror. It has options like `--all`, `--installed`, and `--verbose` to display all components, installed components, and component versions respectively. The output includes component information and version information based on the specified component. --- # tiup list diff --git a/tiup/tiup-command-mirror-clone.md b/tiup/tiup-command-mirror-clone.md index 1ba0725aafd26..dfe19acfbdea4 100644 --- a/tiup/tiup-command-mirror-clone.md +++ b/tiup/tiup-command-mirror-clone.md @@ -1,5 +1,6 @@ --- title: tiup mirror clone +summary: The `tiup mirror clone` command is used to clone an existing mirror or its components to create a new mirror with a different signature key. The syntax is `tiup mirror clone [global version] [flags]`. Options include cloning the whole mirror, specifying platform and operating system, and matching component versions by prefix. --- # tiup mirror clone diff --git a/tiup/tiup-command-mirror-genkey.md b/tiup/tiup-command-mirror-genkey.md index 18739fbe308f0..16ffc54deaaea 100644 --- a/tiup/tiup-command-mirror-genkey.md +++ b/tiup/tiup-command-mirror-genkey.md @@ -1,5 +1,6 @@ --- title: tiup mirror genkey +summary: TiUP mirror genkey is a command used to generate a private key for TiUP. It has options to specify the name of the key and to show the corresponding public key. The command also allows saving the public key information as a file. It is important not to transmit private keys over the Internet. --- # tiup mirror genkey diff --git a/tiup/tiup-command-mirror-grant.md b/tiup/tiup-command-mirror-grant.md index 8f3ad9a959310..3930b426a1cfa 100644 --- a/tiup/tiup-command-mirror-grant.md +++ b/tiup/tiup-command-mirror-grant.md @@ -1,5 +1,6 @@ --- title: tiup mirror grant +summary: The `tiup mirror grant` command is used to add a component owner to the current mirror. The owner needs to send their public key to the mirror administrator before being added. This command is only supported for local mirrors. The syntax is `tiup mirror grant `. The options include specifying the key and name of the component owner. If successful, there is no output. If there are errors, TiUP reports the specific error. --- # tiup mirror grant diff --git a/tiup/tiup-command-mirror-init.md b/tiup/tiup-command-mirror-init.md index 7d5f48088a577..a77dfcac015e2 100644 --- a/tiup/tiup-command-mirror-init.md +++ b/tiup/tiup-command-mirror-init.md @@ -1,5 +1,6 @@ --- title: tiup mirror init +summary: The `tiup mirror init` command initializes an empty mirror, generating root.json, 1.index.json, snapshot.json, and timestamp.json files. Use `tiup mirror init ` to specify a local directory for mirror files. Use the -k or --key-dir option to specify the directory for private key files. If the specified directory is not empty, an error will be reported. --- # tiup mirror init diff --git a/tiup/tiup-command-mirror-merge.md b/tiup/tiup-command-mirror-merge.md index ca16e2225c57f..96321f89e68e5 100644 --- a/tiup/tiup-command-mirror-merge.md +++ b/tiup/tiup-command-mirror-merge.md @@ -1,5 +1,6 @@ --- title: tiup mirror merge +summary: The `tiup mirror merge` command merges one or more mirrors to the current mirror. Conditions for execution include existing owner IDs and corresponding private keys. --- # tiup mirror merge diff --git a/tiup/tiup-command-mirror-modify.md b/tiup/tiup-command-mirror-modify.md index d9553f4398017..7a4d71dee5911 100644 --- a/tiup/tiup-command-mirror-modify.md +++ b/tiup/tiup-command-mirror-modify.md @@ -1,5 +1,6 @@ --- title: tiup mirror modify +summary: The tiup mirror modify command is used to modify published components. Only valid component owners can modify their published components. The syntax is "tiup mirror modify [version] [flags]". Options include -k, --yank, --hide, and --standalone. If the command is executed successfully, there is no output. If the component owner is not authorized to modify the target component, TiUP reports an error. --- # tiup mirror modify diff --git a/tiup/tiup-command-mirror-publish.md b/tiup/tiup-command-mirror-publish.md index 50312cf430ea0..ef2022becdc78 100644 --- a/tiup/tiup-command-mirror-publish.md +++ b/tiup/tiup-command-mirror-publish.md @@ -1,5 +1,6 @@ --- title: tiup mirror publish +summary: The `tiup mirror publish` command is used to publish new components or versions. Only component owners with access can publish. --- # tiup mirror publish diff --git a/tiup/tiup-command-mirror-rotate.md b/tiup/tiup-command-mirror-rotate.md index 9359f84abac6b..602f6d6f38101 100644 --- a/tiup/tiup-command-mirror-rotate.md +++ b/tiup/tiup-command-mirror-rotate.md @@ -1,5 +1,6 @@ --- title: tiup mirror rotate +summary: TiUP mirror rotate is used to update the root.json file in a TiUP mirror. It contains public keys, expiration date, and is signed by administrators. The command automates the update process and requires all administrators to sign the file. Before using the command, ensure all TiUP clients are upgraded to v1.5.0 or later. --- # tiup mirror rotate diff --git a/tiup/tiup-command-mirror-set.md b/tiup/tiup-command-mirror-set.md index 2a8a9ba9a79fb..0473202760a18 100644 --- a/tiup/tiup-command-mirror-set.md +++ b/tiup/tiup-command-mirror-set.md @@ -1,5 +1,6 @@ --- title: tiup mirror set +summary: The `tiup mirror set` command switches the current mirror between local file system and remote network address. The official mirror address is `https://tiup-mirrors.pingcap.com`. Use `tiup mirror set ` to set the mirror address. Use `-r, --root` option to specify the root certificate for network mirrors to prevent man-in-the-middle attacks. No output is generated. --- # tiup mirror set diff --git a/tiup/tiup-command-mirror-sign.md b/tiup/tiup-command-mirror-sign.md index 7c5f45657fbb5..012b03b53b722 100644 --- a/tiup/tiup-command-mirror-sign.md +++ b/tiup/tiup-command-mirror-sign.md @@ -1,5 +1,6 @@ --- title: tiup mirror sign +summary: The `tiup mirror sign` command is used to sign metadata files in TiUP mirror. It supports network addresses and local file paths. Options include specifying the private key location and setting the access timeout for network signing. Successful execution results in no output, while errors are reported for duplicate signing or invalid manifest files. --- # tiup mirror sign diff --git a/tiup/tiup-command-mirror.md b/tiup/tiup-command-mirror.md index a534d5bd1dbb3..e66c3863dc9c8 100644 --- a/tiup/tiup-command-mirror.md +++ b/tiup/tiup-command-mirror.md @@ -1,5 +1,6 @@ --- title: tiup mirror +summary: TiUP mirror is a crucial concept in TiUP, supporting local and remote mirroring. The 'tiup mirror' command manages mirrors, creating, distributing components, and managing keys. The syntax is 'tiup mirror [flags]'. Supported sub-commands include genkey, sign, init, set, grant, publish, modify, rotate, clone, and merge. --- # tiup mirror diff --git a/tiup/tiup-command-status.md b/tiup/tiup-command-status.md index 2e41389f59ee6..55c86182d8e5d 100644 --- a/tiup/tiup-command-status.md +++ b/tiup/tiup-command-status.md @@ -1,5 +1,6 @@ --- title: tiup status +summary: The "tiup status" command is used to view the operation information of the components after running them using the "tiup " command. It shows the name, component, PID, status, created time, directory, binary, and arguments of the operating components. The component status can be Up, Down, Tombstone, Pending Offline, or Unknown. The status is derived from the PD scheduling information. --- # tiup status diff --git a/tiup/tiup-command-telemetry.md b/tiup/tiup-command-telemetry.md index 2218b0145b260..e01cafb99c218 100644 --- a/tiup/tiup-command-telemetry.md +++ b/tiup/tiup-command-telemetry.md @@ -1,5 +1,6 @@ --- title: tiup telemetry +summary: TiUP telemetry is now disabled by default in v1.11.3. Usage information is not collected or shared with PingCAP. When enabled, it shares telemetry identifiers and command execution status. It does not share cluster details. Use 'tiup telemetry' command to control telemetry with sub-commands like status, reset, enable, and disable. --- # tiup telemetry diff --git a/tiup/tiup-command-uninstall.md b/tiup/tiup-command-uninstall.md index ad71761195d75..ad299a16926aa 100644 --- a/tiup/tiup-command-uninstall.md +++ b/tiup/tiup-command-uninstall.md @@ -1,5 +1,6 @@ --- title: tiup uninstall +summary: The tiup uninstall command is used to uninstall installed components. It has options to uninstall all versions of a component and to uninstall TiUP itself. If the command exits without error, it outputs "Uninstalled component successfully!" If no version or --all is specified, it reports an error to use "tiup uninstall tidbx --all" to remove all versions. --- # tiup uninstall diff --git a/tiup/tiup-command-update.md b/tiup/tiup-command-update.md index 9cbc2a3bfab34..70b2abd93d208 100644 --- a/tiup/tiup-command-update.md +++ b/tiup/tiup-command-update.md @@ -1,5 +1,6 @@ --- title: tiup update +summary: The tiup update command is used to update installed components or TiUP itself. You can specify the component and version to update, use options like --all, --force, --nightly, or --self, and receive outputs for successful updates or unsupported versions. --- # tiup update diff --git a/tiup/tiup-component-cluster-audit-cleanup.md b/tiup/tiup-component-cluster-audit-cleanup.md index 33191207632e5..fc9e7fae5f98b 100644 --- a/tiup/tiup-component-cluster-audit-cleanup.md +++ b/tiup/tiup-component-cluster-audit-cleanup.md @@ -1,5 +1,6 @@ --- title: tiup cluster audit cleanup +summary: The `tiup cluster audit cleanup` command is used to clean up logs generated by the `tiup cluster` command. It has options to specify the number of days logs are retained and to print help information. The output confirms successful log cleaning. --- # tiup cluster audit cleanup diff --git a/tiup/tiup-component-cluster-audit.md b/tiup/tiup-component-cluster-audit.md index 44419f2bf9024..ff02253a71e30 100644 --- a/tiup/tiup-component-cluster-audit.md +++ b/tiup/tiup-component-cluster-audit.md @@ -1,5 +1,6 @@ --- title: tiup cluster audit +summary: The tiup cluster audit command is used to view commands executed on all clusters in the history and the execution log of each command. If [audit-id] is specified, the corresponding execution log is output. If not specified, a table with fields ID, Time, and Command is output in reverse chronological order. The -h, --help option prints help information and is disabled by default. --- # tiup cluster audit diff --git a/tiup/tiup-component-cluster-check.md b/tiup/tiup-component-cluster-check.md index f187a4038d8a5..02f557ce0d1c1 100644 --- a/tiup/tiup-component-cluster-check.md +++ b/tiup/tiup-component-cluster-check.md @@ -1,5 +1,6 @@ --- title: tiup cluster check +summary: TiUP Cluster provides a `check` command to ensure hardware and software environments meet production requirements. It checks OS version, CPU support, time synchronization, system limits, and more. Options include automatic repair and enabling checks for CPU core number, memory size, and disk performance. Use `tiup cluster check [flags]` command to perform checks. Use `--apply` to attempt automatic repair. Use `-N, --node` and `-R, --role` to specify nodes and roles to check. Use `--enable-cpu`, `--enable-disk`, and `--enable-mem` to enable specific checks. --- # tiup cluster check @@ -10,7 +11,7 @@ For a formal production environment, before the environment goes live, you need ### Operating system version -Check the operating system distribution and version of the deployed machines. Currently, only CentOS 7 is supported for deployment. More system versions may be supported in later releases for compatibility improvement. +Check the operating system distribution and version of the deployed machines. For a list of supported versions, see [OS and platform requirements](/hardware-and-software-requirements.md#os-and-platform-requirements). ### CPU EPOLLEXCLUSIVE @@ -18,7 +19,7 @@ Check whether the CPU of the target machine supports EPOLLEXCLUSIVE. ### numactl -Check whether numactl is installed on the target machine. If tied cores are configured on the target machine, you must install numactl. +Check whether `numactl` is installed on the target machine. If tied cores are configured on the target machine, you must install `numactl`. ### System time @@ -65,7 +66,7 @@ Check the limit values in the `/etc/security/limits.conf` file: ### SELinux -Check whether SELinux is enabled. It is recommended to disable SELinux. +Check whether SELinux is enabled. It is required to disable SELinux. ### Firewall diff --git a/tiup/tiup-component-cluster-clean.md b/tiup/tiup-component-cluster-clean.md index 47eec3b1f3250..c3d927d44e1ee 100644 --- a/tiup/tiup-component-cluster-clean.md +++ b/tiup/tiup-component-cluster-clean.md @@ -1,5 +1,6 @@ --- title: tiup cluster clean +summary: The `tiup cluster clean` command is used to reset a cluster in a test environment by stopping the cluster and deleting all data. It has options to clean data, logs, or both, and can ignore specific nodes or roles. Use with caution in a production environment. --- # tiup cluster clean diff --git a/tiup/tiup-component-cluster-deploy.md b/tiup/tiup-component-cluster-deploy.md index 83dea8af58841..51df660bc8848 100644 --- a/tiup/tiup-component-cluster-deploy.md +++ b/tiup/tiup-component-cluster-deploy.md @@ -1,5 +1,6 @@ --- title: tiup cluster deploy +summary: The tiup cluster deploy command is used to deploy a new cluster with specified options such as cluster name, version, and topology file. Additional options include user, identity file, password, ignore config check, skip labels, skip create user, and help. The output is the deployment log. --- # tiup cluster deploy @@ -13,7 +14,7 @@ tiup cluster deploy [flags] ``` - ``: the name of the new cluster, which cannot be the same as the existing cluster names. -- ``: the version number of the TiDB cluster to deploy, such as `v7.4.0`. +- ``: the version number of the TiDB cluster to deploy, such as `v{{{ .tidb-version }}}`. - ``: the prepared [topology file](/tiup/tiup-cluster-topology-reference.md). ## Options diff --git a/tiup/tiup-component-cluster-destroy.md b/tiup/tiup-component-cluster-destroy.md index cb69edf8d6417..ef193d0ef8aa8 100644 --- a/tiup/tiup-component-cluster-destroy.md +++ b/tiup/tiup-component-cluster-destroy.md @@ -1,5 +1,6 @@ --- title: tiup cluster destroy +summary: The tiup cluster destroy command stops the cluster and deletes log, deployment, and data directories for each service. It also deletes parent directories created by tiup-cluster. Options include --force to ignore errors, --retain-node-data to specify nodes to retain data, --retain-role-data to specify roles to retain data, and -h or --help to print help information. The output is the execution log of the tiup-cluster. --- # tiup cluster destroy diff --git a/tiup/tiup-component-cluster-disable.md b/tiup/tiup-component-cluster-disable.md index 765fe5423a7e1..d690332ae9200 100644 --- a/tiup/tiup-component-cluster-disable.md +++ b/tiup/tiup-component-cluster-disable.md @@ -1,5 +1,6 @@ --- title: tiup cluster disable +summary: The `tiup cluster disable` command is used to disable the auto-enabling of cluster service after restarting the machine. It executes `systemctl disable ` on the specified node. Options include -N for specifying nodes and -R for specifying roles. The output is the execution log of the tiup-cluster. --- # tiup cluster disable diff --git a/tiup/tiup-component-cluster-display.md b/tiup/tiup-component-cluster-display.md index 64e8365155860..4d4124aa532c0 100644 --- a/tiup/tiup-component-cluster-display.md +++ b/tiup/tiup-component-cluster-display.md @@ -1,5 +1,6 @@ --- title: tiup cluster display +summary: tiup cluster display command efficiently shows the operation status of each component in the cluster. It provides options to display dashboard information, node status, CPU and memory usage, and more. The output includes cluster name, version, SSH client type, dashboard address, and a table with node details. Node service status can be Up, Down, Tombstone, Pending Offline, or Unknown. --- # tiup cluster display diff --git a/tiup/tiup-component-cluster-edit-config.md b/tiup/tiup-component-cluster-edit-config.md index ef7b6780396a8..26a638246aa94 100644 --- a/tiup/tiup-component-cluster-edit-config.md +++ b/tiup/tiup-component-cluster-edit-config.md @@ -1,5 +1,6 @@ --- title: tiup cluster edit-config +summary: The `tiup cluster edit-config` command allows you to modify the cluster configuration after deployment. You can use an editor to modify the topology file, specified in the `$EDITOR` environment variable. Note that you cannot add or delete machines when modifying the configuration. After executing the command, the configuration is modified only on the control machine, and you need to execute `tiup cluster reload` to reload the configuration. --- # tiup cluster edit-config diff --git a/tiup/tiup-component-cluster-enable.md b/tiup/tiup-component-cluster-enable.md index 094a0c5ab46c9..6a8e73cfff776 100644 --- a/tiup/tiup-component-cluster-enable.md +++ b/tiup/tiup-component-cluster-enable.md @@ -1,5 +1,6 @@ --- title: tiup cluster enable +summary: The `tiup cluster enable` command is used to automatically enable cluster services after a machine restart. It executes `systemctl enable ` at the specified node. Options include specifying nodes or roles for auto-enabling, and the `-h, --help` option prints help information. The output is the execution log of the tiup-cluster. --- # tiup cluster enable diff --git a/tiup/tiup-component-cluster-help.md b/tiup/tiup-component-cluster-help.md index 8c0e1f1f0edaa..4349016534f4e 100644 --- a/tiup/tiup-component-cluster-help.md +++ b/tiup/tiup-component-cluster-help.md @@ -1,5 +1,6 @@ --- title: tiup cluster help +summary: tiup-cluster provides help information for users in the command line interface. Use the `help` command or `--help` option to access it. Specify `[command]` to view help information for a specific command. The output is the help information of the specified command or tiup-cluster. --- # tiup cluster help diff --git a/tiup/tiup-component-cluster-import.md b/tiup/tiup-component-cluster-import.md index bd2fedc7fb34b..a30273ff65c00 100644 --- a/tiup/tiup-component-cluster-import.md +++ b/tiup/tiup-component-cluster-import.md @@ -1,5 +1,6 @@ --- title: tiup cluster import +summary: TiUP Cluster provides the `import` command to transfer TiDB clusters from TiDB Ansible to TiUP for management. Do not use `import` for clusters with certain configurations. Use options like `--dir` and `--rename` to customize the import process. --- # tiup cluster import diff --git a/tiup/tiup-component-cluster-list.md b/tiup/tiup-component-cluster-list.md index 8e694bb08802a..7a5f727d2a43d 100644 --- a/tiup/tiup-component-cluster-list.md +++ b/tiup/tiup-component-cluster-list.md @@ -1,5 +1,6 @@ --- title: tiup cluster list +summary: tiup-cluster supports deploying multiple clusters using the same control machine. The `tiup cluster list` command outputs all clusters deployed by the currently logged-in user. The deployed cluster data is stored in the `~/.tiup/storage/cluster/clusters/` directory. Users can view the cluster name, deployment user, version, path, and private key used to connect the cluster. --- # tiup cluster list diff --git a/tiup/tiup-component-cluster-meta-backup.md b/tiup/tiup-component-cluster-meta-backup.md index 741b9f276bf73..5e82befe81e5f 100644 --- a/tiup/tiup-component-cluster-meta-backup.md +++ b/tiup/tiup-component-cluster-meta-backup.md @@ -1,5 +1,6 @@ --- title: tiup cluster meta backup +summary: The TiUP meta file is crucial for cluster operation and maintenance. Use `tiup cluster meta backup` to regularly back up the file. Use `tiup dm list` to check the cluster name. Specify the target directory with `--file` option. Use `-h, --help` for help information. The output includes execution logs of tiup-cluster. --- # tiup cluster meta backup @@ -12,7 +13,7 @@ The TiUP meta file is used for cluster operation and maintenance (OM). If this f tiup cluster meta backup [flags] ``` -`` is the name of the cluster to be operated on. If you forget the cluster name, you can check it using the [`tiup dm list`](/tiup/tiup-component-dm-list.md) command. +`` is the name of the cluster to be operated on. If you forget the cluster name, you can check it using the [`tiup cluster list`](/tiup/tiup-component-cluster-list.md) command. ## Options diff --git a/tiup/tiup-component-cluster-meta-restore.md b/tiup/tiup-component-cluster-meta-restore.md index 155563981a7bb..eae6d62243613 100644 --- a/tiup/tiup-component-cluster-meta-restore.md +++ b/tiup/tiup-component-cluster-meta-restore.md @@ -1,5 +1,6 @@ --- title: tiup cluster meta restore +summary: To restore the TiUP meta file, use the `tiup cluster meta restore` command with cluster name and backup file path. The restore operation overwrites the current meta file, so it should only be done when the file is lost. The `-h` or `--help` option prints help information. The output includes the execution logs of tiup-cluster. --- # tiup cluster meta restore diff --git a/tiup/tiup-component-cluster-patch.md b/tiup/tiup-component-cluster-patch.md index efd3338d472d5..24d8d9ad45169 100644 --- a/tiup/tiup-component-cluster-patch.md +++ b/tiup/tiup-component-cluster-patch.md @@ -1,5 +1,6 @@ --- title: tiup cluster patch +summary: The `tiup cluster patch` command allows for dynamic replacement of binaries in a running cluster. It uploads the binary package, stops the target service, replaces the binary, and starts the service. Preparation involves packing the binary package and using options like `--overwrite`, `--transfer-timeout`, `-N, --node`, `-R, --role`, and `--offline`. The output is the execution log of the tiup-cluster. --- # tiup cluster patch @@ -28,7 +29,7 @@ Before running the `tiup cluster patch` command, you need to pack the binary pac 1. Determine the following variables: - `${component}`: the name of the component to be replaced (such as `tidb`, `tikv`, or `pd`). - - `${version}`: the version of the component (such as `v7.4.0` or `v6.5.3`). + - `${version}`: the version of the component (such as `v{{{ .tidb-version }}}`). - `${os}`: the operating system (`linux`). - `${arch}`: the platform on which the component runs (`amd64`, `arm64`). diff --git a/tiup/tiup-component-cluster-prune.md b/tiup/tiup-component-cluster-prune.md index 1940338781815..91aa09a0ab670 100644 --- a/tiup/tiup-component-cluster-prune.md +++ b/tiup/tiup-component-cluster-prune.md @@ -1,5 +1,6 @@ --- title: tiup cluster prune +summary: When scaling in the cluster, TiUP does not immediately stop services or delete data for some components. You must wait for data scheduling to complete and then manually execute the 'tiup cluster prune' command to clean up. The syntax is 'tiup cluster prune [flags]'. The option '-h, --help' prints help information and the output is the log of the cleanup process. --- # tiup cluster prune diff --git a/tiup/tiup-component-cluster-reload.md b/tiup/tiup-component-cluster-reload.md index b7c0f288eb4ac..da540b492a99f 100644 --- a/tiup/tiup-component-cluster-reload.md +++ b/tiup/tiup-component-cluster-reload.md @@ -1,5 +1,6 @@ --- title: tiup cluster reload +summary: The `tiup cluster reload` command is used to apply modified cluster configurations and restart services. It can be forced with `--force`, set a transfer timeout with `--transfer-timeout`, ignore config checks with `--ignore-config-check`, specify nodes with `-N, --node`, roles with `-R, --role`, and skip restart with `--skip-restart`. The output is the execution log of the tiup-cluster. --- # tiup cluster reload diff --git a/tiup/tiup-component-cluster-rename.md b/tiup/tiup-component-cluster-rename.md index 79e7e3a4c0f33..7dcd6300303c7 100644 --- a/tiup/tiup-component-cluster-rename.md +++ b/tiup/tiup-component-cluster-rename.md @@ -1,5 +1,6 @@ --- title: tiup cluster rename +summary: The `tiup cluster rename` command is used to change the cluster name after it has been deployed. Additional steps are required if the `dashboard_dir` field of `grafana_servers` is configured for the TiUP cluster. The syntax for the command is `tiup cluster rename `. The `-h, --help` option prints help information. The output is the execution log of the tiup-cluster. --- # tiup cluster rename diff --git a/tiup/tiup-component-cluster-replay.md b/tiup/tiup-component-cluster-replay.md index cfa134c6794e3..96266e48e1091 100644 --- a/tiup/tiup-component-cluster-replay.md +++ b/tiup/tiup-component-cluster-replay.md @@ -1,5 +1,6 @@ --- title: tiup cluster replay +summary: The `tiup cluster replay` command allows you to retry failed cluster operations and skip successfully performed steps. Use `tiup cluster replay ` to retry the command with the specified audit ID. View audit IDs with `tiup cluster audit` command. The output is the result of the specified audit ID. --- # tiup cluster replay diff --git a/tiup/tiup-component-cluster-restart.md b/tiup/tiup-component-cluster-restart.md index 5bd9a994d3202..194f53348c732 100644 --- a/tiup/tiup-component-cluster-restart.md +++ b/tiup/tiup-component-cluster-restart.md @@ -1,5 +1,6 @@ --- title: tiup cluster restart +summary: The `tiup cluster restart` command is used to restart services in a specified cluster. During the restart, the services are unavailable. You can specify nodes or roles to be restarted using the `-N, --node` and `-R, --role` options. The output is the log of the service restart process. --- # tiup cluster restart diff --git a/tiup/tiup-component-cluster-scale-in.md b/tiup/tiup-component-cluster-scale-in.md index 3be6b8068d8d4..bbd56b4ce8665 100644 --- a/tiup/tiup-component-cluster-scale-in.md +++ b/tiup/tiup-component-cluster-scale-in.md @@ -1,5 +1,6 @@ --- title: tiup cluster scale-in +summary: The `tiup cluster scale-in` command is used to scale in the cluster by taking specified nodes offline, removing them from the cluster, and deleting remaining files. Components like TiKV, TiFlash, and TiDB Binlog are handled asynchronously and require additional steps to check and clean up. The command also includes options for node specification, forceful removal, transfer timeout, and help information. --- # tiup cluster scale-in diff --git a/tiup/tiup-component-cluster-scale-out.md b/tiup/tiup-component-cluster-scale-out.md index fba6627d53551..23c31858b35e5 100644 --- a/tiup/tiup-component-cluster-scale-out.md +++ b/tiup/tiup-component-cluster-scale-out.md @@ -1,5 +1,6 @@ --- title: tiup cluster scale-out +summary: The tiup cluster scale-out command is used to add new nodes to the cluster. It establishes an SSH connection to the new node, creates necessary directories, and updates the configuration. Options include -u for user, -i for identity file, -p for password, --no-labels to skip label check, --skip-create-user to skip user check, and -h for help. The output is the log of scaling out. --- # tiup cluster scale-out diff --git a/tiup/tiup-component-cluster-start.md b/tiup/tiup-component-cluster-start.md index 3edf69019d0f3..b9871d657d12e 100644 --- a/tiup/tiup-component-cluster-start.md +++ b/tiup/tiup-component-cluster-start.md @@ -1,5 +1,6 @@ --- title: tiup cluster start +summary: The tiup cluster start command is used to start all or some services of a specified cluster. It has options like --init for safe start, -N for specifying nodes, -R for specifying roles, and -h for help. The output is the log of starting the service. --- # tiup cluster start diff --git a/tiup/tiup-component-cluster-stop.md b/tiup/tiup-component-cluster-stop.md index f8192e8ef02a7..77a49b7636e6c 100644 --- a/tiup/tiup-component-cluster-stop.md +++ b/tiup/tiup-component-cluster-stop.md @@ -1,5 +1,6 @@ --- title: tiup cluster stop +summary: The "tiup cluster stop" command is used to stop all or some services of a specified cluster. If the core services are stopped, the cluster cannot provide services anymore. The command syntax is "tiup cluster stop [flags]". Options include -N/--node to specify nodes to be stopped, -R/--role to specify roles of nodes to be stopped, and -h/--help to print help information. The output is the log of stopping the service. --- # tiup cluster stop diff --git a/tiup/tiup-component-cluster-template.md b/tiup/tiup-component-cluster-template.md index 628002ccaee56..a1c96a01be657 100644 --- a/tiup/tiup-component-cluster-template.md +++ b/tiup/tiup-component-cluster-template.md @@ -1,5 +1,6 @@ --- title: tiup cluster template +summary: The tiup cluster template command is used to prepare a topology file for cluster deployment. It has options to output default, detailed, local, or multi-dc topology templates. The output can be redirected to the topology file for deployment. --- # tiup cluster template @@ -16,7 +17,8 @@ If this option is not specified, the output default template contains the follow - 3 PD instances - 3 TiKV instances -- 1 TiDB instance +- 3 TiDB instances +- 2 TiFlash instances - 1 Prometheus instance - 1 Grafana instance - 1 Alertmanager instance @@ -28,6 +30,11 @@ If this option is not specified, the output default template contains the follow - Outputs a detailed topology template that is commented with configurable parameters. To enable this option, add it to the command. - If this option is not specified, the simple topology template is output by default. +### --local + +- Outputs a simple topology template for the local cluster, which can be used directly, and the `global` parameter can be adjusted as needed. +- This template creates a PD service, a TiDB service, a TiKV service, a Prometheus service, and a Grafana service. + ### --multi-dc - Outputs the topology template of multiple data centers. To enable this option, add it to the command. diff --git a/tiup/tiup-component-cluster-tls.md b/tiup/tiup-component-cluster-tls.md new file mode 100644 index 0000000000000..d7f5b48f98165 --- /dev/null +++ b/tiup/tiup-component-cluster-tls.md @@ -0,0 +1,53 @@ +--- +title: tiup cluster tls +summary: The `tiup cluster tls` command is used to enable or disable TLS (Transport Layer Security) between cluster components. +--- + +# tiup cluster tls + +The `tiup cluster tls` command is used to enable TLS (Transport Layer Security) between cluster components. It automatically generates and distributes self-signed certificates to each node in the cluster. + +## Syntax + +```shell +tiup cluster tls [flags] +``` + +`` specifies the cluster for which you want to enable or disable TLS. + +> **Note:** +> +> Currently, the `tiup cluster tls` command supports enabling or disabling TLS only for clusters with a single PD node. For clusters with multiple PD nodes, executing the `tiup cluster tls` command directly returns an error, because switching the TLS status can cause communication exceptions between PD nodes. To enable or disable TLS for a cluster with multiple PD nodes, first [`scale-in`](/tiup/tiup-component-cluster-scale-in.md) the PD nodes to one node, and then execute the `tiup cluster tls` command. + +## Options + +### --clean-certificate + +- When you disable TLS, use this option to clean up previously generated certificates. +- Data type: `BOOLEAN` +- Default: `false` +- If you do not specify this option, old certificates might be reused when you enable TLS again. + +### --force + +- Forces enabling or disabling TLS, regardless of the cluster's current TLS status. +- Data type: `BOOLEAN` +- Default: `false` +- If you do not specify this option, the operation is skipped if the cluster is already in the requested state. + +### --reload-certificate + +- When you enable TLS, use this option to regenerate certificates. +- Data type: `BOOLEAN` +- Default: `false` +- If you do not specify this option, new certificates are not generated if certificates already exist. + +### -h, --help + +- Prints the help information. +- Data type: `BOOLEAN` +- Default: `false` + +## Output + +Execution logs of the tiup-cluster command. diff --git a/tiup/tiup-component-cluster-upgrade.md b/tiup/tiup-component-cluster-upgrade.md index a07be6f19c4ef..a53707582c8ac 100644 --- a/tiup/tiup-component-cluster-upgrade.md +++ b/tiup/tiup-component-cluster-upgrade.md @@ -1,5 +1,6 @@ --- title: tiup cluster upgrade +summary: The tiup cluster upgrade command is used to upgrade a specified cluster to a specific version. It requires the cluster name and target version as input. Options include --force to ignore errors and start the cluster, --transfer-timeout to set maximum wait time for node migration, --ignore-config-check to skip configuration check, and --offline to replace binary files without restarting the cluster. The output is the log of the upgrading progress. --- # tiup cluster upgrade @@ -13,7 +14,7 @@ tiup cluster upgrade [flags] ``` - ``: the cluster name to operate on. If you forget the cluster name, you can check it with the [cluster list](/tiup/tiup-component-cluster-list.md) command. -- ``: the target version to upgrade to, such as `v7.4.0`. Currently, it is only allowed to upgrade to a version higher than the current cluster, that is, no downgrade is allowed. It is also not allowed to upgrade to the nightly version. +- ``: the target version to upgrade to, such as `v{{{ .tidb-version }}}`. Currently, it is only allowed to upgrade to a version higher than the current cluster, that is, no downgrade is allowed. It is also not allowed to upgrade to the nightly version. ## Options @@ -49,6 +50,73 @@ tiup cluster upgrade [flags] - Data type: `BOOLEAN` - This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. +### --pd-version + +- Specifies the version of PD. If this option is set, the version of PD will no longer be consistent with the cluster version. +- Data type: `STRINGS` +- If this option is not set, the version of PD remains consistent with the cluster version. + +### --tikv-version + +- Specifies the version of TiKV. If this option is set, the version of TiKV will no longer be consistent with the cluster version. +- Data type: `STRINGS` +- If this option is not set, the version of TiKV remains consistent with the cluster version. + +### --tikv-cdc-version + +- Specifies the version of TiKV CDC. If this option is set, the version of TiKV CDC will no longer be consistent with the cluster version. +- Data type: `STRINGS` +- If this option is not set, the version of TiKV CDC remains consistent with the cluster version. + +### --tiflash-version + +- Specifies the version of TiFlash. If this option is set, the version of TiFlash will no longer be consistent with the cluster version. +- Data type: `STRINGS` +- If this option is not set, the version of TiFlash remains consistent with the cluster version. + +### --cdc-version + +- Specifies the version of TiCDC. If this option is set, the version of TiCDC will no longer be consistent with the cluster version. +- Data type: `STRINGS` +- If this option is not set, the version of TiCDC remains consistent with the cluster version. + +### --tiproxy-version + +- Specifies the version of TiProxy. If this option is set, the version of TiProxy will no longer be consistent with the cluster version. +- Data type: `STRINGS` +- If this option is not set, the version of TiProxy remains consistent with the cluster version. + +### --tidb-dashboard-version + +- Specifies the version of TiDB Dashboard. If this option is set, the version of TiDB Dashboard will no longer be consistent with the cluster version. +- Data type: `STRINGS` +- If this option is not set, the version of TiDB Dashboard remains consistent with the cluster version. + +### --alertmanager-version + +- Specifies the version of alert manager. If this option is set, the version of alert manager will no longer be consistent with the cluster version. +- Data type: `STRINGS` +- If this option is not set, the version of alert manager remains consistent with the cluster version. + +### --blackbox-exporter-version + +- Specifies the version of Blackbox Exporter. If this option is set, the version of Blackbox Exporter will no longer be consistent with the cluster version. +- Data type: `STRINGS` +- If this option is not set, the version of Blackbox Exporter remains consistent with the cluster version. + +### --node-exporter-version + +- Specifies the version of Node Exporter. If this option is set, the version of Node Exporter will no longer be consistent with the cluster version. +- Data type: `STRINGS` +- If this option is not set, the version of Node Exporter remains consistent with the cluster version. + +### --restart-timeout + +- Specifies the waiting time after upgrading a component during a rolling upgrade. +- Data type: `STRINGS`. Any type that can be parsed by [`golang time.ParseDuration`](https://pkg.go.dev/time#ParseDuration) is supported. +- Default: `0` +- If this option is not specified, there will be no waiting time after upgrading a component. + ### -h, --help - Prints the help information. diff --git a/tiup/tiup-component-cluster.md b/tiup/tiup-component-cluster.md index 0c4bf2f0ce78e..940ae3932c783 100644 --- a/tiup/tiup-component-cluster.md +++ b/tiup/tiup-component-cluster.md @@ -1,5 +1,6 @@ --- title: TiUP Cluster +summary: TiUP Cluster is a cluster management component of TiUP written in Golang. It is used for daily operations and maintenance, including deploying, starting, shutting down, destroying, elastic scaling, upgrading TiDB clusters, and managing TiDB cluster parameters. The syntax for using TiUP Cluster is "tiup cluster [command] [flags]". Supported commands include import, template, check, deploy, list, display, start, stop, restart, scale-in, scale-out, upgrade, prune, edit-config, reload, patch, rename, clean, destroy, audit, replay, enable, disable, meta backup, meta restore, and help. --- # TiUP Cluster diff --git a/tiup/tiup-component-dm-audit.md b/tiup/tiup-component-dm-audit.md index f30f7d957b478..a209382137053 100644 --- a/tiup/tiup-component-dm-audit.md +++ b/tiup/tiup-component-dm-audit.md @@ -1,5 +1,6 @@ --- title: tiup dm audit +summary: The `tiup dm audit` command is used to view historical commands executed on all clusters and the execution log of each command. If `[audit-id]` is not filled, the table of operation records is output in reverse chronological order, showing the `audit-id`, execution time, and command. If `[audit-id]` is filled, the execution log of the specified `audit-id` is checked. The `-h, --help` option prints help information. If `[audit-id]` is specified, the corresponding execution log is output. If not specified, a table with the fields ID, Time, and Command is output. --- # tiup dm audit diff --git a/tiup/tiup-component-dm-deploy.md b/tiup/tiup-component-dm-deploy.md index 3ef4b0f3b5ca1..b0b15f2540b97 100644 --- a/tiup/tiup-component-dm-deploy.md +++ b/tiup/tiup-component-dm-deploy.md @@ -1,5 +1,6 @@ --- title: tiup dm deploy +summary: The `tiup dm deploy` command is used to deploy a new cluster. It requires the cluster name, version, and a prepared topology file. Optional flags include user, identity file, password, and help. The output is the deployment log. --- # tiup dm deploy diff --git a/tiup/tiup-component-dm-destroy.md b/tiup/tiup-component-dm-destroy.md index 0c7f04d4097aa..9a547c343e1f3 100644 --- a/tiup/tiup-component-dm-destroy.md +++ b/tiup/tiup-component-dm-destroy.md @@ -1,5 +1,6 @@ --- title: tiup dm destroy +summary: The `tiup dm destroy` command stops the cluster, deletes log, deployment, and data directories for each service, and also deletes parent directories created by `tiup-dm`. The syntax is `tiup dm destroy [flags]`. The option `-h, --help` prints help information. The output is the execution log of tiup-dm. --- # tiup dm destroy diff --git a/tiup/tiup-component-dm-disable.md b/tiup/tiup-component-dm-disable.md index fca46d3ae5c42..1e363028c4290 100644 --- a/tiup/tiup-component-dm-disable.md +++ b/tiup/tiup-component-dm-disable.md @@ -1,5 +1,6 @@ --- title: tiup dm disable +summary: The `tiup dm disable` command is used to disable the auto-enabling of cluster service after restarting the machine. It can be used with options like `-N, --node` to specify nodes and `-R, --role` to specify roles for disabling auto-enabling. The output is the execution log of the tiup-dm command. --- # tiup dm disable diff --git a/tiup/tiup-component-dm-display.md b/tiup/tiup-component-dm-display.md index 4d20cfef25b53..22f2355c7e39d 100644 --- a/tiup/tiup-component-dm-display.md +++ b/tiup/tiup-component-dm-display.md @@ -1,5 +1,6 @@ --- title: tiup dm display +summary: tiup dm display command efficiently checks the operational status of each component in a DM cluster. It requires the cluster name and can also specify node IDs and roles. The output includes cluster name, version, SSH client type, and a table with fields like ID, Role, Host, Ports, OS/Arch, Status, Data Dir, and Deploy Dir. --- # tiup dm display diff --git a/tiup/tiup-component-dm-edit-config.md b/tiup/tiup-component-dm-edit-config.md index 30f48a84fb86b..9a948052da0f5 100644 --- a/tiup/tiup-component-dm-edit-config.md +++ b/tiup/tiup-component-dm-edit-config.md @@ -1,5 +1,6 @@ --- title: tiup dm edit-config +summary: The `tiup dm edit-config` command allows you to modify the cluster service configuration after deployment. You can use an editor to modify the topology file of the specified cluster. Note that you cannot add or delete machines when modifying the configuration. After executing the command, the configuration is modified only on the control machine, and you need to execute the `tiup dm reload` command to reload the configuration. --- # tiup dm edit-config diff --git a/tiup/tiup-component-dm-enable.md b/tiup/tiup-component-dm-enable.md index f16000128a843..c95e140d45b9d 100644 --- a/tiup/tiup-component-dm-enable.md +++ b/tiup/tiup-component-dm-enable.md @@ -1,5 +1,6 @@ --- title: tiup dm enable +summary: The `tiup dm enable` command is used to enable the auto-enabling of cluster services after a machine restart. It executes `systemctl enable ` at the specified node. Options include specifying nodes or roles for auto-enabling. The output is the execution log of tiup-dm. --- # tiup dm enable diff --git a/tiup/tiup-component-dm-help.md b/tiup/tiup-component-dm-help.md index 035530c224aae..43ef7dadde728 100644 --- a/tiup/tiup-component-dm-help.md +++ b/tiup/tiup-component-dm-help.md @@ -1,5 +1,6 @@ --- title: tiup dm help +summary: The tiup-dm command-line interface provides a wealth of help information, which can be accessed using the `help` command or the `--help` option. The syntax for accessing help is `tiup dm help [command] [flags]`, where `[command]` specifies the command for which help information is needed. The `-h` or `--help` option prints the help information. The output is the help information for the specified command or `tiup-dm`. --- # tiup dm help diff --git a/tiup/tiup-component-dm-import.md b/tiup/tiup-component-dm-import.md index d238adec21c1b..f8dc2debc6871 100644 --- a/tiup/tiup-component-dm-import.md +++ b/tiup/tiup-component-dm-import.md @@ -1,5 +1,6 @@ --- title: tiup dm import +summary: The `import` command in TiUP DM is used to upgrade DM clusters from v1.0 to v2.0 or later versions. It does not support importing DM Portal components from v1.0 clusters and requires stopping the original cluster before importing. The command only supports importing to DM v2.0.0-rc.2 and later versions and can be used to import a DM v1.0 cluster to a new DM v2.0 cluster. After importing, there is only one DM-master node in the cluster, and the deployment directories of some components might be different from those in the original cluster. --- # tiup dm import Only for upgrading DM v1.0 diff --git a/tiup/tiup-component-dm-list.md b/tiup/tiup-component-dm-list.md index c7719d5999b5a..db1c21072582b 100644 --- a/tiup/tiup-component-dm-list.md +++ b/tiup/tiup-component-dm-list.md @@ -1,5 +1,6 @@ --- title: tiup dm list +summary: tiup-dm supports deploying multiple clusters using the same control machine. The "tiup dm list" command checks which clusters are deployed by the currently logged-in user. The data is stored in the ~/.tiup/storage/dm/clusters/ directory. The user can view the cluster name, deploying user, version, path, and private key. --- # tiup dm list diff --git a/tiup/tiup-component-dm-prune.md b/tiup/tiup-component-dm-prune.md index 66fef792543d7..afcc21a254924 100644 --- a/tiup/tiup-component-dm-prune.md +++ b/tiup/tiup-component-dm-prune.md @@ -1,5 +1,6 @@ --- title: tiup dm prune +summary: When scaling in the cluster, a small amount of metadata in etcd may not be cleaned up, but it usually doesn't cause any problems. If needed, you can manually execute the "tiup dm prune" command to clean up the metadata. The command syntax is "tiup dm prune [flags]". The option "-h, --help" prints help information and the output is the log of the cleanup process. --- # tiup dm prune diff --git a/tiup/tiup-component-dm-reload.md b/tiup/tiup-component-dm-reload.md index 7fda1d8186925..ef63c760b3820 100644 --- a/tiup/tiup-component-dm-reload.md +++ b/tiup/tiup-component-dm-reload.md @@ -1,5 +1,6 @@ --- title: tiup dm reload +summary: The `tiup dm reload` command is used to apply modified cluster configurations and restart the service. It can specify nodes and roles to be restarted, or skip the restart process. The command also provides an option to print help information and outputs the execution log of tiup-dm. --- # tiup dm reload diff --git a/tiup/tiup-component-dm-replay.md b/tiup/tiup-component-dm-replay.md index 7a15f900f413f..ed1cedeccc9e0 100644 --- a/tiup/tiup-component-dm-replay.md +++ b/tiup/tiup-component-dm-replay.md @@ -1,5 +1,6 @@ --- title: tiup dm replay +summary: The `tiup dm replay` command allows you to retry failed cluster operations and skip successfully performed steps. Use the `audit-id` of the command to be retried, which can be found using the `tiup dm audit` command. This helps save time when re-performing operations in a large cluster. --- # tiup dm replay diff --git a/tiup/tiup-component-dm-restart.md b/tiup/tiup-component-dm-restart.md index 6910fae1455ae..c4f76e71a2a1f 100644 --- a/tiup/tiup-component-dm-restart.md +++ b/tiup/tiup-component-dm-restart.md @@ -1,5 +1,6 @@ --- title: tiup dm restart +summary: The `tiup dm restart` command is used to restart services in a specified cluster. During the restart, the services are unavailable. The syntax is `tiup dm restart [flags]`. Options include -N for specifying nodes to restart, -R for specifying roles of nodes to restart, and -h for help information. The output is the log of the service restart process. --- # tiup dm restart diff --git a/tiup/tiup-component-dm-scale-in.md b/tiup/tiup-component-dm-scale-in.md index af3b3c1a022f0..2e053e3e186cd 100644 --- a/tiup/tiup-component-dm-scale-in.md +++ b/tiup/tiup-component-dm-scale-in.md @@ -1,5 +1,6 @@ --- title: tiup dm scale-in +summary: The tiup dm scale-in command is used to scale in the cluster by taking the service offline and removing the specified node from the cluster. The syntax is "tiup dm scale-in [flags]". Options include -N, --force, and -h for specifying nodes, forcing removal of down nodes, and printing help information. The output is the log of scaling in. --- # tiup dm scale-in diff --git a/tiup/tiup-component-dm-scale-out.md b/tiup/tiup-component-dm-scale-out.md index da77f96e94812..cb2cbff23d233 100644 --- a/tiup/tiup-component-dm-scale-out.md +++ b/tiup/tiup-component-dm-scale-out.md @@ -1,5 +1,6 @@ --- title: tiup dm scale-out +summary: The `tiup dm scale-out` command is used to scale out the cluster by establishing an SSH connection to the new node, creating necessary directories, deploying, and starting the service. --- # tiup dm scale-out diff --git a/tiup/tiup-component-dm-start.md b/tiup/tiup-component-dm-start.md index 4a5c9ff1acf2b..b3dd35854b2ba 100644 --- a/tiup/tiup-component-dm-start.md +++ b/tiup/tiup-component-dm-start.md @@ -1,5 +1,6 @@ --- title: tiup dm start +summary: The tiup dm start command is used to start services of a specified cluster. The syntax is "tiup dm start [flags]". Options include -N/--node to specify nodes, -R/--role to specify roles, and -h/--help to print help information. The output is the log of starting the service. --- # tiup dm start diff --git a/tiup/tiup-component-dm-stop.md b/tiup/tiup-component-dm-stop.md index 72cd009d4e733..16aab4446c59d 100644 --- a/tiup/tiup-component-dm-stop.md +++ b/tiup/tiup-component-dm-stop.md @@ -1,5 +1,6 @@ --- title: tiup dm stop +summary: The `tiup dm stop` command is used to stop services in a specified cluster. You can specify nodes and roles to be stopped using the `-N, --node` and `-R, --role` options. The output is the log of stopping the service. --- # tiup dm stop diff --git a/tiup/tiup-component-dm-template.md b/tiup/tiup-component-dm-template.md index eedd9c492fd91..6a6d2c80de2f7 100644 --- a/tiup/tiup-component-dm-template.md +++ b/tiup/tiup-component-dm-template.md @@ -1,5 +1,6 @@ --- title: tiup dm template +summary: TiUP DM template command is used to output the built-in topology file template for cluster deployment. The default template includes 3 DM-master instances, 3 DM-worker instances, 1 Prometheus instance, 1 Grafana instance, and 1 Alertmanager instance. The --full option outputs a detailed topology template with configurable parameters. The output can be redirected to the topology file for deployment. --- # tiup dm template diff --git a/tiup/tiup-component-dm-upgrade.md b/tiup/tiup-component-dm-upgrade.md index 75a3a0ae93d73..56dd306c27bff 100644 --- a/tiup/tiup-component-dm-upgrade.md +++ b/tiup/tiup-component-dm-upgrade.md @@ -1,5 +1,6 @@ --- title: tiup dm upgrade +summary: The `tiup dm upgrade` command upgrades a specified cluster to a specific version. It requires the cluster name and target version as parameters. The `--offline` option allows for offline upgrades, and the `-h, --help` option prints help information. The output is a log of the service upgrade process. --- # tiup dm upgrade @@ -13,7 +14,7 @@ tiup dm upgrade [flags] ``` - `` is the name of the cluster to be operated on. If you forget the cluster name, you can check it using the [`tiup dm list`](/tiup/tiup-component-dm-list.md) command. -- `` is the target version to be upgraded to, such as `v7.4.0`. Currently, only upgrading to a later version is allowed, and upgrading to an earlier version is not allowed, which means the downgrade is not allowed. Upgrading to a nightly version is not allowed either. +- `` is the target version to be upgraded to, such as `v{{{ .tidb-version }}}`. Currently, only upgrading to a later version is allowed, and upgrading to an earlier version is not allowed, which means the downgrade is not allowed. Upgrading to a nightly version is not allowed either. ## Options diff --git a/tiup/tiup-component-dm.md b/tiup/tiup-component-dm.md index f162b08c76df3..f543de0c8fa7b 100644 --- a/tiup/tiup-component-dm.md +++ b/tiup/tiup-component-dm.md @@ -1,5 +1,6 @@ --- title: TiUP DM +summary: TiUP DM is used to manage DM clusters, including deploying, starting, stopping, destroying, scaling, upgrading, and managing configuration parameters. It supports options like SSH, timeout, confirmation skipping, version printing, and help information. Supported commands include import, template, deploy, list, display, start, stop, restart, scale-in, scale-out, upgrade, prune, edit-config, reload, patch, destroy, audit, replay, enable, disable, and help. --- # TiUP DM diff --git a/tiup/tiup-component-management.md b/tiup/tiup-component-management.md index 3a9136dba566f..1741c7bebc856 100644 --- a/tiup/tiup-component-management.md +++ b/tiup/tiup-component-management.md @@ -1,7 +1,6 @@ --- title: Manage TiUP Components with TiUP Commands summary: Learn how to manage TiUP components using TiUP commands. -aliases: ['/tidb/dev/manage-tiup-component','/docs/dev/tiup/manage-tiup-component/','/docs/dev/reference/tools/tiup/manage-component/'] --- # Manage TiUP Components with TiUP Commands @@ -15,6 +14,8 @@ You can use the following TiUP commands to manage components in the TiUP ecosyst - status: Checks the status of a running component. - clean: Cleans up the instance on which a component is deployed. - help: Prints the help information. If you append another TiUP command to this command, the usage of the appended command is printed. +- link: Links the binary of the component to the executable directory (`$TIUP_HOME/bin/`). +- unlink: Deletes the soft link generated by the `tiup link` command. This document introduces the common component management operations and the corresponding TiUP commands. @@ -70,12 +71,12 @@ Example 2: Use TiUP to install the nightly version of TiDB. tiup install tidb:nightly ``` -Example 3: Use TiUP to install TiKV v7.4.0. +Example 3: Use TiUP to install TiKV v{{{ .tidb-version }}}. {{< copyable "shell-regular" >}} ```shell -tiup install tikv:v7.4.0 +tiup install tikv:v{{{ .tidb-version }}} ``` ## Upgrade components @@ -128,12 +129,12 @@ Before the component is started, TiUP creates a directory for it, and then puts If you want to start the same component multiple times and reuse the previous working directory, you can use `--tag` to specify the same name when the component is started. After the tag is specified, the working directory will *not be automatically deleted* when the instance is terminated, which makes it convenient to reuse the working directory. -Example 1: Operate TiDB v7.4.0. +Example 1: Operate TiDB v{{{ .tidb-version }}}. {{< copyable "shell-regular" >}} ```shell -tiup tidb:v7.4.0 +tiup tidb:v{{{ .tidb-version }}} ``` Example 2: Specify the tag with which TiKV operates. @@ -219,12 +220,12 @@ The following flags are supported in this command: - If the version is ignored, adding `--all` means to uninstall all versions of this component. - If the version and the component are both ignored, adding `--all` means to uninstall all components of all versions. -Example 1: Uninstall TiDB v7.4.0. +Example 1: Uninstall TiDB v{{{ .tidb-version }}}. {{< copyable "shell-regular" >}} ```shell -tiup uninstall tidb:v7.4.0 +tiup uninstall tidb:v{{{ .tidb-version }}} ``` Example 2: Uninstall TiKV of all versions. @@ -242,3 +243,43 @@ Example 3: Uninstall all installed components. ```shell tiup uninstall --all ``` + +### Link components + +TiUP v1.13.0 adds the experimental `link` and `unlink` commands to link the binary of a component to the executable directory (`$TIUP_HOME/bin/`) and remove the link. This feature enables you to call a component without going through TiUP each time, while preserving the ability to switch between different versions. However, this method lacks the process of automatic update checking and setting certain environment variables and some components (such as ctl) cannot be used. Therefore, it is recommended to use it only when necessary. + +Example 1: install and link the latest version of the cluster component + +```shell +tiup install cluster +tiup link cluster +``` + +Example 2: switch the cluster component to version v1.13.0 + +```shell +tiup link cluster:v1.13.0 +``` + +The following is the output of the `tiup link cluster` command: + +```shell +package cluster provides these executables: tiup-cluster +``` + +This indicates that the binary name of the cluster component is `tiup-cluster`. After the link command is completed, you can use the cluster component by directly typing `tiup-cluster` into the command line. + +Example 3: cancel the link of the cluster component + +```shell +tiup unlink cluster +``` + +Example 4: manage the version of TiUP + +Before v1.13.0, TiUP is installed in `~/.tiup/bin/` and different versions cannot coexist. Starting from v1.13.0, you can install and link TiUP like any other components. + +```shell +tiup update --self # update tiup itself to a version that supports link +tiup link tiup:v1.13.0 +``` diff --git a/tiup/tiup-dm-topology-reference.md b/tiup/tiup-dm-topology-reference.md index 6bb6cbe71e895..7ba2d8254bae3 100644 --- a/tiup/tiup-dm-topology-reference.md +++ b/tiup/tiup-dm-topology-reference.md @@ -1,5 +1,6 @@ --- title: Topology Configuration File for DM Cluster Deployment Using TiUP +summary: To deploy or scale a TiDB Data Migration (DM) cluster using TiUP, a topology file is required to describe the cluster's global configuration, server configurations, master servers, worker servers, monitoring servers, Grafana servers, and Alertmanager servers. Each section contains specific fields for configuration. The topology file structure includes global, server_configs, master_servers, worker_servers, monitoring_servers, grafana_servers, and alertmanager_servers. Each section has its own set of configurable fields for deployment and configuration. --- # Topology Configuration File for DM Cluster Deployment Using TiUP diff --git a/tiup/tiup-documentation-guide.md b/tiup/tiup-documentation-guide.md index 5a1b2ae51c70d..2b36a33bc8333 100644 --- a/tiup/tiup-documentation-guide.md +++ b/tiup/tiup-documentation-guide.md @@ -1,7 +1,6 @@ --- title: TiUP Documentation Map summary: Guide you through TiUP documentation with links and introductions. -aliases: ['/docs/dev/tiup/tiup-documentation-guide/'] --- # TiUP Documentation Map diff --git a/tiup/tiup-faq.md b/tiup/tiup-faq.md index 6d423d13dd2d0..ab55eaa6ed81b 100644 --- a/tiup/tiup-faq.md +++ b/tiup/tiup-faq.md @@ -1,7 +1,6 @@ --- title: TiUP FAQs summary: Provide answers to common questions asked by TiUP users. -aliases: ['/docs/dev/tiup/tiup-faq/'] --- # TiUP FAQs @@ -23,7 +22,7 @@ TiUP does not support third-party components for the time being, but the TiUP Te ## What is the difference between the TiUP playground and TiUP cluster components? -The TiUP playground component is mainly used to build a stand-alone development environment on Linux or macOS operating systems. It helps you get started quickly and run a specified version of the TiUP cluster easily. The TiUP cluster component is mainly used to deploy and maintain a production environment cluster, which is usually a large-scale cluster. +The TiUP playground component is mainly used to build a stand-alone development environment on Linux or macOS operating systems. It helps you get started quickly and run a specified version of the TiUP cluster easily. The TiUP cluster component is mainly used to deploy and maintain a production environment cluster, which is usually a large-scale cluster. TiDB clusters deployed by TiUP Playground might lack some features and operational capabilities, and it is not recommended for complete functional and stability testing. ## How do I write the topology file for the TiUP cluster component? diff --git a/tiup/tiup-mirror-reference.md b/tiup/tiup-mirror-reference.md index 4eb30cd2fcb14..520c6d50232d3 100644 --- a/tiup/tiup-mirror-reference.md +++ b/tiup/tiup-mirror-reference.md @@ -154,7 +154,7 @@ The index file's format is as follows: } }, "name": "{owner-name}", # The name of the owner. - "threshod": {N} # Indicates that the components owned by the owner must have at least N valid signatures. + "threshold": {N} # Indicates that the components owned by the owner must have at least N valid signatures. }, ... "{ownerN}": { # The ID of the Nth owner. diff --git a/tiup/tiup-mirror.md b/tiup/tiup-mirror.md index 485d4f41783ff..62ae69bf67e57 100644 --- a/tiup/tiup-mirror.md +++ b/tiup/tiup-mirror.md @@ -1,7 +1,6 @@ --- title: Create a Private Mirror summary: Learn how to create a private mirror. -aliases: ['/tidb/dev/tiup-mirrors','/docs/dev/tiup/tiup-mirrors/','/docs/dev/reference/tools/tiup/mirrors/'] --- # Create a Private Mirror @@ -87,9 +86,9 @@ The `tiup mirror clone` command provides many optional flags (might provide more If you want to clone only one version (not all versions) of a component, use `--=` to specify this version. For example: - - Execute the `tiup mirror clone --tidb v7.4.0` command to clone the v7.4.0 version of the TiDB component. - - Run the `tiup mirror clone --tidb v7.4.0 --tikv all` command to clone the v7.4.0 version of the TiDB component and all versions of the TiKV component. - - Run the `tiup mirror clone v7.4.0` command to clone the v7.4.0 version of all components in a cluster. + - Execute the `tiup mirror clone --tidb v{{{ .tidb-version }}}` command to clone the v{{{ .tidb-version }}} version of the TiDB component. + - Run the `tiup mirror clone --tidb v{{{ .tidb-version }}} --tikv all` command to clone the v{{{ .tidb-version }}} version of the TiDB component and all versions of the TiKV component. + - Run the `tiup mirror clone v{{{ .tidb-version }}}` command to clone the v{{{ .tidb-version }}} version of all components in a cluster. After cloning, signing keys are set up automatically. diff --git a/tiup/tiup-overview.md b/tiup/tiup-overview.md index eeb0243a427bb..f26e8ad0c7ef0 100644 --- a/tiup/tiup-overview.md +++ b/tiup/tiup-overview.md @@ -1,7 +1,6 @@ --- title: TiUP Overview summary: Introduce the TiUP tool and its ecosystem. -aliases: ['/docs/dev/tiup/tiup-overview/','/docs/dev/reference/tools/tiup/overview/'] --- # TiUP Overview @@ -28,6 +27,13 @@ After installation, you can check the version of TiUP: tiup --version ``` +```bash +1.14.0 tiup +Go Version: go1.21.4 +Git Ref: v1.14.0 +GitHash: c3e9fc518aea0da66a37f82ee5a516171de9c372 +``` + > **Note:** > > For TiUP versions starting from v1.11.3, the telemetry is disabled by default in newly deployed TiUP, and usage information is not collected and shared with PingCAP. For details about what is shared and how to disable the sharing, see [Telemetry](/telemetry.md). @@ -56,6 +62,19 @@ the latest stable version will be downloaded from the repository. Usage: tiup [flags] [args...] tiup [flags] [args...] + tiup [command] + +Examples: + $ tiup playground # Quick start + $ tiup playground nightly # Start a playground with the latest nightly version + $ tiup install [:version] # Install a component of specific version + $ tiup update --all # Update all installed components to the latest version + $ tiup update --nightly # Update all installed components to the nightly version + $ tiup update --self # Update the "tiup" to the latest version + $ tiup list # Fetch the latest supported components list + $ tiup status # Display all running/terminated instances + $ tiup clean # Clean the data of running/terminated instance (Kill process if it's running) + $ tiup clean --all # Clean the data of all running/terminated instances Available Commands: install Install a specific version of a component @@ -65,33 +84,21 @@ Available Commands: status List the status of instantiated components clean Clean the data of instantiated components mirror Manage a repository mirror for TiUP components - help Help about any command or component - -Components Manifest: - use "tiup list" to fetch the latest components manifest + telemetry Controls things about telemetry + env Show the list of system environment variable that related to TiUP + history Display the historical execution record of TiUP, displays 100 lines by default + link Link component binary to $TIUP_HOME/bin/ + unlink Unlink component binary to $TIUP_HOME/bin/ + help Help about any command + completion Generate the autocompletion script for the specified shell Flags: --binary [:version] Print binary path of a specific version of a component [:version] and the latest version installed will be selected if no version specified --binpath string Specify the binary path of component instance -h, --help help for tiup - -T, --tag string Specify a tag for component instance - -v, --version version for tiup - -Component instances with the same "tag" will share a data directory ($TIUP_HOME/data/$tag): - $ tiup --tag mycluster playground - -Examples: - $ tiup playground # Quick start - $ tiup playground nightly # Start a playground with the latest nightly version - $ tiup install [:version] # Install a component of specific version - $ tiup update --all # Update all installed components to the latest version - $ tiup update --nightly # Update all installed components to the nightly version - $ tiup update --self # Update the "tiup" to the latest version - $ tiup list # Fetch the latest supported components list - $ tiup status # Display all running/terminated instances - $ tiup clean # Clean the data of running/terminated instance (Kill process if it's running) - $ tiup clean --all # Clean the data of all running/terminated instances + -T, --tag string [Deprecated] Specify a tag for component instance + -v, --version Print the version of tiup Use "tiup [command] --help" for more information about a command. ``` @@ -99,17 +106,23 @@ Use "tiup [command] --help" for more information about a command. The output is long but you can focus on only two parts: - Available commands - - install: used to install components - - list: used to view the list of available components - - uninstall: used to uninstall components + - install: used to install a specific version of a component + - list: used to view the list of available TiDB components or available versions of a component + - uninstall: used to uninstall components or versions of a component - update: used to update the component version - status: used to view the running history of components - clean: used to clear the running log of components - mirror: used to clone a private mirror from the official mirror + - telemetry: used to control the telemetry feature + - env: used to show the list of system environment variables that related to TiUP + - history: used to display the historical execution record of TiUP (100 lines by default) + - link: used to link component binary to $TIUP_HOME/bin/ + - unlink: used to unlink component binary to $TIUP_HOME/bin/ - help: used to print out help information + - completion: used to generate the command line autocompletion script for the specified shell (including bash, zsh, fish, and powershell) - Available components - playground: used to start a TiDB cluster locally - - client: used to connect to a TiDB cluster in a local machine + - client: client used to connect to TiUP Playground - cluster: used to deploy a TiDB cluster for production environments - bench: used to stress test the database diff --git a/tiup/tiup-playground.md b/tiup/tiup-playground.md index c15395c2b1492..1d5f2a9f308b4 100644 --- a/tiup/tiup-playground.md +++ b/tiup/tiup-playground.md @@ -1,7 +1,6 @@ --- title: Quickly Deploy a Local TiDB Cluster summary: Learn how to quickly deploy a local TiDB cluster using the playground component of TiUP. -aliases: ['/docs/dev/tiup/tiup-playground/','/docs/dev/reference/tools/tiup/playground/'] --- # Quickly Deploy a Local TiDB Cluster @@ -22,7 +21,7 @@ This command actually performs the following operations: - Because this command does not specify the version of the playground component, TiUP first checks the latest version of the installed playground component. Assume that the latest version is v1.12.3, then this command works the same as `tiup playground:v1.12.3`. - If you have not used TiUP playground to install the TiDB, TiKV, and PD components, the playground component installs the latest stable version of these components, and then start these instances. -- Because this command does not specify the version of the TiDB, PD, and TiKV component, TiUP playground uses the latest version of each component by default. Assume that the latest version is v7.4.0, then this command works the same as `tiup playground:v1.12.3 v7.4.0`. +- Because this command does not specify the version of the TiDB, PD, and TiKV component, TiUP playground uses the latest version of each component by default. Assume that the latest version is v{{{ .tidb-version }}}, then this command works the same as `tiup playground:v1.12.3 v{{{ .tidb-version }}}`. - Because this command does not specify the number of each component, TiUP playground, by default, starts a smallest cluster that consists of one TiDB instance, one TiKV instance, one PD instance, and one TiFlash instance. - After starting each TiDB component, TiUP playground reminds you that the cluster is successfully started and provides you some useful information, such as how to connect to the TiDB cluster through the MySQL client and how to access the [TiDB Dashboard](/dashboard/dashboard-intro.md). @@ -90,7 +89,7 @@ In the command above, `nightly` indicates the latest development version of TiDB ### Override PD's default configuration -First, you need to copy the [PD configuration template](https://github.com/pingcap/pd/blob/master/conf/config.toml). Assume you place the copied file to `~/config/pd.toml` and make some changes according to your need, then you can execute the following command to override PD's default configuration: +First, you need to copy the [PD configuration template](https://github.com/pingcap/pd/blob/release-7.5/conf/config.toml). Assume you place the copied file to `~/config/pd.toml` and make some changes according to your need, then you can execute the following command to override PD's default configuration: ```shell tiup playground --pd.config ~/config/pd.toml diff --git a/tiup/tiup-reference.md b/tiup/tiup-reference.md index d286315ecb9f9..2b22b07914337 100644 --- a/tiup/tiup-reference.md +++ b/tiup/tiup-reference.md @@ -1,5 +1,6 @@ --- title: TiUP Reference +summary: TiUP is the package manager for the TiDB ecosystem, managing components like TiDB, PD, and TiKV. It supports commands like install, list, uninstall, update, status, clean, mirror, telemetry, completion, env, and help. It also manages the cluster and TiDB Data Migration (DM) cluster. --- # TiUP Reference diff --git a/tiup/tiup-terminology-and-concepts.md b/tiup/tiup-terminology-and-concepts.md index a44de21943c36..7ff8d201ffaec 100644 --- a/tiup/tiup-terminology-and-concepts.md +++ b/tiup/tiup-terminology-and-concepts.md @@ -1,7 +1,6 @@ --- title: TiUP Terminology and Concepts summary: Explain the terms and concepts of TiUP. -aliases: ['/docs/dev/tiup/tiup-terminology-and-concepts/'] --- # TiUP Terminology and Concepts diff --git a/tiup/tiup-troubleshooting-guide.md b/tiup/tiup-troubleshooting-guide.md index be2ca01c79f95..f6d225ed952bf 100644 --- a/tiup/tiup-troubleshooting-guide.md +++ b/tiup/tiup-troubleshooting-guide.md @@ -1,7 +1,6 @@ --- title: TiUP Troubleshooting Guide summary: Introduce the troubleshooting methods and solutions if you encounter issues when using TiUP. -aliases: ['/docs/dev/tiup/tiup-troubleshooting-guide/'] --- # TiUP Troubleshooting Guide diff --git a/transaction-isolation-levels.md b/transaction-isolation-levels.md index 1a78c24e6e4f4..9a5c9966f8b04 100644 --- a/transaction-isolation-levels.md +++ b/transaction-isolation-levels.md @@ -1,7 +1,6 @@ --- title: TiDB Transaction Isolation Levels summary: Learn about the transaction isolation levels in TiDB. -aliases: ['/docs/dev/transaction-isolation-levels/','/docs/dev/reference/transactions/transaction-isolation/'] --- # TiDB Transaction Isolation Levels @@ -77,7 +76,7 @@ Starting from v6.0.0, TiDB supports using the [`tidb_rc_read_check_ts`](/system- - If TiDB does not encounter any data update during the read process, it returns the result to the client and the `SELECT` statement is successfully executed. - If TiDB encounters data update during the read process: - If TiDB has not yet sent the result to the client, TiDB tries to acquire a new timestamp and retry this statement. - - If TiDB has already sent partial data to the client, TiDB reports an error to the client. The amount of data sent to the client each time is controlled by `tidb_init_chunk_size` and `tidb_max_chunk_size`. + - If TiDB has already sent partial data to the client, TiDB reports an error to the client. The amount of data sent to the client each time is controlled by [`tidb_init_chunk_size`](/system-variables.md#tidb_init_chunk_size) and [`tidb_max_chunk_size`](/system-variables.md#tidb_max_chunk_size). In scenarios where the `READ-COMMITTED` isolation level is used, the `SELECT` statements are many, and read-write conflicts are rare, enabling this variable can avoid the latency and cost of getting the global timestamp. diff --git a/transaction-overview.md b/transaction-overview.md index e12e891f22dcf..07da0672ddaaa 100644 --- a/transaction-overview.md +++ b/transaction-overview.md @@ -1,7 +1,6 @@ --- title: Transactions summary: Learn transactions in TiDB. -aliases: ['/docs/dev/transaction-overview/','/docs/dev/reference/transactions/overview/'] --- # Transactions diff --git a/troubleshoot-high-disk-io.md b/troubleshoot-high-disk-io.md index 997750736894a..1ef2c2f1d6b84 100644 --- a/troubleshoot-high-disk-io.md +++ b/troubleshoot-high-disk-io.md @@ -52,7 +52,7 @@ In addition, some other panel metrics might help you determine whether the bottl - Whether the value of `store-pool-size` of `[raftstore]` is too small. It is recommended to set this value between `[1,5]` and not too large. - Whether the CPU resource of the machine is insufficient. -- `append log` is slow. TiKV Grafana's `Raft I/O` and `append log duration` metrics are relatively high, which might usually occur along with relatively high `Raft Propose`/`apply wait duration`. The possible causes are as follows: +- `apply log` is slow. TiKV Grafana's `Raft I/O` and `apply log duration` metrics are relatively high, which might usually occur along with relatively high `Raft Propose`/`apply wait duration`. The possible causes are as follows: - The value of `apply-pool-size` of `[raftstore]` is too small. It is recommended to set this value between `[1, 5]` and not too large. The value of `Thread CPU`/`apply cpu` is also relatively high. - Insufficient CPU resources on the machine. @@ -74,7 +74,7 @@ In addition, some other panel metrics might help you determine whether the bottl If the disk's I/O capability fails to keep up with the write, it is recommended to scale up the disk. If the throughput of the disk reaches the upper limit (for example, the throughput of SATA SSD is much lower than that of NVMe SSD), which results in write stall, but the CPU resource is relatively sufficient, you can try to use a compression algorithm of higher compression ratio to relieve the pressure on the disk, that is, use CPU resources to make up for disk resources. - For example, when the pressure of `default cf compaction` is relatively high, you can change the parameter`[rocksdb.defaultcf] compression-per-level = ["no", "no", "lz4", "lz4", "lz4", "zstd" , "zstd"]` to `compression-per-level = ["no", "no", "zstd", "zstd", "zstd", "zstd", "zstd"]`. + For example, when the pressure of `default cf compaction` is relatively high, you can change the parameter`[rocksdb.defaultcf] compression-per-level = ["no", "no", "lz4", "lz4", "lz4", "zstd", "zstd"]` to `compression-per-level = ["no", "no", "zstd", "zstd", "zstd", "zstd", "zstd"]`. ### I/O issues found in alerts diff --git a/troubleshoot-lock-conflicts.md b/troubleshoot-lock-conflicts.md index 7e6a978d0ca74..3fd13ca1ab491 100644 --- a/troubleshoot-lock-conflicts.md +++ b/troubleshoot-lock-conflicts.md @@ -86,13 +86,14 @@ For example, to filter transactions with a long lock-waiting time using the `whe {{< copyable "sql" >}} ```sql -select trx.* from information_schema.data_lock_waits as l left join information_schema.tidb_trx as trx on l.trx_id = trx.id where l.key = "7480000000000000415F728000000000000001"\G +select trx.* from information_schema.data_lock_waits as l left join information_schema.cluster_tidb_trx as trx on l.trx_id = trx.id where l.key = "7480000000000000415F728000000000000001"\G ``` The following is an example output: ```sql *************************** 1. row *************************** + INSTANCE: 127.0.0.1:10080 ID: 426831815660273668 START_TIME: 2021-08-06 07:16:00.081000 CURRENT_SQL_DIGEST: 06da614b93e62713bd282d4685fc5b88d688337f36e88fe55871726ce0eb80d7 @@ -106,6 +107,7 @@ CURRENT_SQL_DIGEST_TEXT: update `t` set `v` = `v` + ? where `id` = ? ; DB: test ALL_SQL_DIGESTS: ["0fdc781f19da1c6078c9de7eadef8a307889c001e05f107847bee4cfc8f3cdf3","06da614b93e62713bd282d4685fc5b88d688337f36e88fe55871726ce0eb80d7"] *************************** 2. row *************************** + INSTANCE: 127.0.0.1:10080 ID: 426831818019569665 START_TIME: 2021-08-06 07:16:09.081000 CURRENT_SQL_DIGEST: 06da614b93e62713bd282d4685fc5b88d688337f36e88fe55871726ce0eb80d7 @@ -324,7 +326,7 @@ Solutions: ### TTL manager has timed out -The transaction execution time cannot exceed the GC time limit. In addition, the TTL time of pessimistic transactions has an upper limit, whose default value is 1 hour. Therefore, a pessimistic transaction executed for more than 1 hour will fail to commit. This timeout threshold is controlled by the TiDB parameter [`performance.max-txn-ttl`](https://github.com/pingcap/tidb/blob/master/pkg/config/config.toml.example). +The transaction execution time cannot exceed the GC time limit. In addition, the TTL time of pessimistic transactions has an upper limit, whose default value is 1 hour. Therefore, a pessimistic transaction executed for more than 1 hour will fail to commit. This timeout threshold is controlled by the TiDB parameter [`performance.max-txn-ttl`](https://github.com/pingcap/tidb/blob/release-7.5/pkg/config/config.toml.example). When the execution time of a pessimistic transaction exceeds the TTL time, the following error message occurs in the TiDB log: diff --git a/troubleshoot-tidb-cluster.md b/troubleshoot-tidb-cluster.md index c43a975b77ab2..f7ea1742b1e01 100644 --- a/troubleshoot-tidb-cluster.md +++ b/troubleshoot-tidb-cluster.md @@ -1,7 +1,6 @@ --- title: TiDB Cluster Troubleshooting Guide summary: Learn how to diagnose and resolve issues when you use TiDB. -aliases: ['/docs/dev/troubleshoot-tidb-cluster/','/docs/dev/how-to/troubleshoot/cluster-setup/'] --- # TiDB Cluster Troubleshooting Guide diff --git a/troubleshoot-tidb-oom.md b/troubleshoot-tidb-oom.md index 95c72e114a943..1c2f102cd0681 100644 --- a/troubleshoot-tidb-oom.md +++ b/troubleshoot-tidb-oom.md @@ -19,7 +19,7 @@ The following are some typical OOM phenomena: - **TiDB-Runtime** > **Memory Usage** shows that the `estimate-inuse` metric keeps rising. - Check `tidb.log`, and you can find the following log entries: - - An alarm about OOM: `[WARN] [memory_usage_alarm.go:139] ["tidb-server has the risk of OOM. Running SQLs and heap profile will be recorded in record path"]`. For more information, see [`memory-usage-alarm-ratio`](/system-variables.md#tidb_memory_usage_alarm_ratio). + - An alarm about OOM: `[WARN] [memory_usage_alarm.go:139] ["tidb-server has the risk of OOM because of memory usage exceeds alarm ratio. Running SQLs and heap profile will be recorded in record path"]`. For more information, see [`memory-usage-alarm-ratio`](/system-variables.md#tidb_memory_usage_alarm_ratio). - A log entry about restart: `[INFO] [printer.go:33] ["Welcome to TiDB."]`. ## Overall troubleshooting process @@ -183,13 +183,13 @@ To locate the root cause of an OOM issue, you need to collect the following info - Run the following command to collect the TiDB Profile information when memory usage is high: ```shell - curl -G http://{TiDBIP}:10080/debug/zip?seconds=10" > profile.zip + curl -G "http://{TiDBIP}:10080/debug/zip?seconds=10" > profile.zip ``` - Run `grep "tidb-server has the risk of OOM" tidb.log` to check the path of the alert file collected by TiDB Server. The following is an example output: ```shell - ["tidb-server has the risk of OOM. Running SQLs and heap profile will be recorded in record path"] ["is tidb_server_memory_limit set"=false] ["system memory total"=14388137984] ["system memory usage"=11897434112] ["tidb-server memory usage"=11223572312] [memory-usage-alarm-ratio=0.8] ["record path"="/tmp/0_tidb/MC4wLjAuMDo0MDAwLzAuMC4wLjA6MTAwODA=/tmp-storage/record"] + ["tidb-server has the risk of OOM because of memory usage exceeds alarm ratio. Running SQLs and heap profile will be recorded in record path"] ["is tidb_server_memory_limit set"=false] ["system memory total"=14388137984] ["system memory usage"=11897434112] ["tidb-server memory usage"=11223572312] [memory-usage-alarm-ratio=0.8] ["record path"="/tmp/0_tidb/MC4wLjAuMDo0MDAwLzAuMC4wLjA6MTAwODA=/tmp-storage/record"] ``` ## See also diff --git a/tso.md b/tso.md index 897006c0120e1..f69a22f832eb0 100644 --- a/tso.md +++ b/tso.md @@ -5,7 +5,7 @@ summary: Learn about TimeStamp Oracle (TSO) in TiDB. # TimeStamp Oracle (TSO) in TiDB -In TiDB, the Placement Driver (PD) plays a pivotal role in allocating timestamps to various components within a cluster. These timestamps are instrumental in the assignment of temporal markers to transactions and data, a mechanism crucial for enabling the [Percolator](https://research.google.com/pubs/pub36726.html) model within TiDB. The Percolator model is used to support Multi-Version Concurrency Control (MVCC) and [transaction management](/transaction-overview.md). +In TiDB, the Placement Driver (PD) plays a pivotal role in allocating timestamps to various components within a cluster. These timestamps are instrumental in the assignment of temporal markers to transactions and data, a mechanism crucial for enabling the [Percolator](https://research.google/pubs/large-scale-incremental-processing-using-distributed-transactions-and-notifications/) model within TiDB. The Percolator model is used to support [Multi-Version Concurrency Control (MVCC)](https://docs.pingcap.com/tidb/stable/glossary#multi-version-concurrency-control-mvcc) and [transaction management](/transaction-overview.md). The following example shows how to get the current TSO in TiDB: diff --git a/tune-operating-system.md b/tune-operating-system.md index ca118bacb0133..5ebe079bdf903 100644 --- a/tune-operating-system.md +++ b/tune-operating-system.md @@ -1,7 +1,6 @@ --- title: Tune Operating System Performance summary: Learn how to tune the parameters of the operating system. -aliases: ['/docs/dev/tune-operating-system/'] --- # Tune Operating System Performance diff --git a/tune-region-performance.md b/tune-region-performance.md index 323cfaa565916..67d0aab505a07 100644 --- a/tune-region-performance.md +++ b/tune-region-performance.md @@ -11,23 +11,21 @@ This document introduces how to tune Region performance by adjusting the Region TiKV automatically [shards bottom-layered data](/best-practices/tidb-best-practices.md#data-sharding). Data is split into multiple Regions based on the key ranges. When the size of a Region exceeds a threshold, TiKV splits it into two or more Regions. -When processing a large amount of data, TiKV might split too many Regions, which causes more resource consumption and [performance regression](/best-practices/massive-regions-best-practices.md#performance-problem). For a certain amount of data, the larger the Region size, the fewer the Regions. Since v6.1.0, TiDB supports setting customizing Region size. The default size of a Region is 96 MiB. To reduce the number of Regions, you can adjust Regions to a larger size. +In scenarios involving large datasets, if the Region size is relatively small, TiKV might have too many Regions, which causes more resource consumption and [performance regression](/best-practices/massive-regions-best-practices.md#performance-problem). Since v6.1.0, TiDB supports customizing Region size. The default size of a Region is 96 MiB. To reduce the number of Regions, you can adjust Regions to a larger size. To reduce the performance overhead of many Regions, you can also enable [Hibernate Region](/best-practices/massive-regions-best-practices.md#method-4-increase-the-number-of-tikv-instances) or [`Region Merge`](/best-practices/massive-regions-best-practices.md#method-5-adjust-raft-base-tick-interval). ## Use `region-split-size` to adjust Region size -> **Warning:** +> **Note:** > -> Currently, customized Region size is an experimental feature introduced in TiDB v6.1.0. It is not recommended that you use it in production environments. The risks are as follows: +> The recommended range for the Region size is [48 MiB, 256 MiB]. Commonly used sizes include 96 MiB, 128 MiB, and 256 MiB. It is NOT recommended to set the Region size beyond 1 GiB. Avoid setting the size to more than 10 GiB. An excessively large Region size might result in the following side effects: > -> + Performance jitter might be caused. -> + The query performance, especially for queries that deal with a large range of data, might decrease. -> + The Region scheduling slows down. - -To adjust the Region size, you can use the [`coprocessor.region-split-size`](/tikv-configuration-file.md#region-split-size) configuration item. The recommended sizes are 96 MiB, 128 MiB, or 256 MiB. The larger the `region-split-size` value, the more jittery the performance will be. It is NOT recommended to set the Region size over 1 GiB. Avoid setting the size to more than 10 GiB. When TiFlash is used, the Region size should not exceed 256 MiB. +> + Performance jitters +> + Decreased query performance, especially for queries that deal with a large range of data +> + Slower Region scheduling -When the Dumpling tool is used, the Region size should not exceed 1 GiB. In this case, you need to reduce the concurrency after increasing the Region size; otherwise, TiDB might run out of memory. +To adjust the Region size, you can use the [`coprocessor.region-split-size`](/tikv-configuration-file.md#region-split-size) configuration item. When TiFlash or the Dumpling tool is used, the Region size should not exceed 1 GiB. After increasing the Region size, you need to reduce the concurrency if the Dumpling tool is used; otherwise, TiDB might run out of memory. ## Use bucket to increase concurrency @@ -35,4 +33,4 @@ When the Dumpling tool is used, the Region size should not exceed 1 GiB. In this > > Currently, this is an experimental feature introduced in TiDB v6.1.0. It is not recommended that you use it in production environments. -When Regions are set to a larger size, you need to set [`coprocessor.enable-region-bucket`](/tikv-configuration-file.md#enable-region-bucket-new-in-v610) to `true` to increase the query concurrency. When you use this configuration, Regions are divided into buckets. Buckets are smaller ranges within a Region and are used as the unit of concurrent query to improve the scan concurrency. You can control the bucket size using [`coprocessor.region-bucket-size`](/tikv-configuration-file.md#region-bucket-size-new-in-v610). \ No newline at end of file +After Regions are set to a larger size, if you want to further improve the query concurrency, you can set [`coprocessor.enable-region-bucket`](/tikv-configuration-file.md#enable-region-bucket-new-in-v610) to `true`. When you use this configuration, Regions are divided into buckets. Buckets are smaller ranges within a Region and are used as the unit of concurrent query to improve the scan concurrency. You can control the bucket size using [`coprocessor.region-bucket-size`](/tikv-configuration-file.md#region-bucket-size-new-in-v610). \ No newline at end of file diff --git a/tune-tikv-memory-performance.md b/tune-tikv-memory-performance.md index 4bd4e9d1ef00e..ce324dd4fe593 100644 --- a/tune-tikv-memory-performance.md +++ b/tune-tikv-memory-performance.md @@ -1,12 +1,11 @@ --- title: Tune TiKV Memory Parameter Performance summary: Learn how to tune the TiKV parameters for optimal performance. -aliases: ['/docs/dev/tune-tikv-performance/','/docs/dev/reference/performance/tune-tikv/','/tidb/dev/tune-tikv-performance'] --- # Tune TiKV Memory Parameter Performance -This document describes how to tune the TiKV parameters for optimal performance. You can find the default configuration file in [etc/config-template.toml](https://github.com/tikv/tikv/blob/master/etc/config-template.toml). To modify the configuration, you can [use TiUP](/maintain-tidb-using-tiup.md#modify-the-configuration) or [modify TiKV dynamically](/dynamic-config.md#modify-tikv-configuration-dynamically) for a limited set of configuration items. For the complete configuration, see [TiKV configuration file](/tikv-configuration-file.md). +This document describes how to tune the TiKV parameters for optimal performance. You can find the default configuration file in [etc/config-template.toml](https://github.com/tikv/tikv/blob/release-7.5/etc/config-template.toml). To modify the configuration, you can [use TiUP](/maintain-tidb-using-tiup.md#modify-the-configuration) or [modify TiKV dynamically](/dynamic-config.md#modify-tikv-configuration-dynamically) for a limited set of configuration items. For the complete configuration, see [TiKV configuration file](/tikv-configuration-file.md). TiKV uses RocksDB for persistent storage at the bottom level of the TiKV architecture. Therefore, many of the performance parameters are related to RocksDB. TiKV uses two RocksDB instances: the default RocksDB instance stores KV data, the Raft RocksDB instance (RaftDB) stores Raft logs. @@ -30,7 +29,7 @@ Each CF also has a separate `write buffer`. You can configure the size by settin ## Parameter specification -``` +```toml # Log level: trace, debug, warn, error, info, off. log-level = "info" @@ -109,7 +108,7 @@ job = "tikv" region-split-check-diff = "32MB" [coprocessor] -## If the size of a Region with the range of [a,e) is larger than the value of `region_max_size`, TiKV trys to split the Region to several Regions, for example, the Regions with the ranges of [a,b), [b,c), [c,d), and [d,e). +## If the size of a Region with the range of [a,e) is larger than the value of `region_max_size`, TiKV tries to split the Region to several Regions, for example, the Regions with the ranges of [a,b), [b,c), [c,d), and [d,e). ## After the Region split, the size of the split Regions is equal to the value of `region_split_size` (or slightly larger than the value of `region_split_size`). # region-max-size = "144MB" # region-split-size = "96MB" diff --git a/tune-tikv-thread-performance.md b/tune-tikv-thread-performance.md index 43a6b4ef98fc1..08371c07814b7 100644 --- a/tune-tikv-thread-performance.md +++ b/tune-tikv-thread-performance.md @@ -1,7 +1,6 @@ --- title: Tune TiKV Thread Pool Performance summary: Learn how to tune TiKV thread pools for optimal performance. -aliases: ['/docs/dev/tune-tikv-thread-performance/'] --- # Tune TiKV Thread Pool Performance diff --git a/two-data-centers-in-one-city-deployment.md b/two-data-centers-in-one-city-deployment.md index 5b08d0261da41..0d3caddf73c57 100644 --- a/two-data-centers-in-one-city-deployment.md +++ b/two-data-centers-in-one-city-deployment.md @@ -1,7 +1,6 @@ --- title: Two Availability Zones in One Region Deployment summary: Learn the deployment solution of two availability zones in one region. -aliases: ['/tidb/dev/synchronous-replication'] --- # Two Availability Zones in One Region Deployment @@ -22,7 +21,7 @@ This section takes the example of a region where two availability zones AZ1 and The architecture of the cluster deployment is as follows: -- The cluster has four replicas: two Voter replicas in AZ1, one Voter replica, and one Learner replica in AZ2. For the TiKV component, each rack has a proper label. +- The cluster has six replicas: three Voter replicas in AZ1, and two Voter replicas along with one Learner replica in AZ2. For the TiKV component, each rack has a proper label. - The Raft protocol is adopted to ensure consistency and high availability of data, which is transparent to users. ![2-AZ-in-1-region architecture](/media/two-dc-replication-1.png) @@ -221,6 +220,7 @@ The replication mode is controlled by PD. You can configure the replication mode primary-replicas = 3 dr-replicas = 2 wait-store-timeout = "1m" + pause-region-split = false ``` - Method 2: If you have deployed a cluster, use pd-ctl commands to modify the configurations of PD. @@ -241,8 +241,9 @@ Descriptions of configuration items: + `replication-mode` is the replication mode to be enabled. In the preceding example, it is set to `dr-auto-sync`. By default, the majority protocol is used. + `label-key` is used to distinguish different AZs and needs to match Placement Rules. In this example, the primary AZ is "east" and the disaster recovery AZ is "west". + `primary-replicas` is the number of Voter replicas in the primary AZ. -+ `dr-replicas` is the number of Voter replicas in the disaster recovery AZ. ++ `dr-replicas` is the number of Voter replicas in the disaster recovery (DR) AZ. + `wait-store-timeout` is the waiting time for switching to asynchronous replication mode when network isolation or failure occurs. If the time of network failure exceeds the waiting time, asynchronous replication mode is enabled. The default waiting time is 60 seconds. ++ `pause-region-split` controls whether to pause Region split operations in the `async_wait` and `async` statuses. Pausing Region split can prevent temporary partial data loss in the DR AZ when synchronizing data in the `sync-recover` status. The default value is `false`. To check the current replication status of the cluster, use the following API: diff --git a/upgrade-monitoring-services.md b/upgrade-monitoring-services.md index 7e93e78aa1fee..cb1bb95337c2a 100644 --- a/upgrade-monitoring-services.md +++ b/upgrade-monitoring-services.md @@ -28,20 +28,29 @@ Download a new installation package from the [Prometheus download page](https:// ### Step 2. Download the Prometheus installation package provided by TiDB -1. Download the TiDB **Server Package** from the [TiDB download page](https://www.pingcap.com/download/) and extract it. -2. In the extracted files, locate `prometheus-v{version}-linux-amd64.tar.gz` and extract it. +1. Download the TiDB server package and extract it. Note that your downloading means you agree to the [Privacy Policy](https://www.pingcap.com/privacy-policy/). + + ``` + https://download.pingcap.com/tidb-community-server-{version}-linux-{arch}.tar.gz + ``` + + > **Tip:** + > + > `{version}` in the link indicates the version number of TiDB and `{arch}` indicates the architecture of the system, which can be `amd64` or `arm64`. For example, the download link for `v{{{ .tidb-version }}}` in the `amd64` architecture is `https://download.pingcap.com/tidb-community-toolkit-v{{{ .tidb-version }}}-linux-amd64.tar.gz`. + +2. In the extracted files, locate `prometheus-v{version}-linux-{arch}.tar.gz` and extract it. ```bash - tar -xzf prometheus-v{version}-linux-amd64.tar.gz + tar -xzf prometheus-v{version}-linux-{arch}.tar.gz ``` ### Step 3. Create a new Prometheus package that TiUP can use -1. Copy the files extracted in [Step 1](#step-1-download-a-new-prometheus-installation-package-from-the-prometheus-website), and then use the copied files to replace the files in the `./prometheus-v{version}-linux-amd64/prometheus` directory extracted in [Step 2](#step-2-download-the-prometheus-installation-package-provided-by-tidb). -2. Recompress the `./prometheus-v{version}-linux-amd64` directory and name the new compressed package as `prometheus-v{new-version}.tar.gz`, where `{new-version}` can be specified according to your need. +1. Copy the files extracted in [Step 1](#step-1-download-a-new-prometheus-installation-package-from-the-prometheus-website), and then use the copied files to replace the files in the `./prometheus-v{version}-linux-{arch}/prometheus` directory extracted in [Step 2](#step-2-download-the-prometheus-installation-package-provided-by-tidb). +2. Recompress the `./prometheus-v{version}-linux-{arch}` directory and name the new compressed package as `prometheus-v{new-version}.tar.gz`, where `{new-version}` can be specified according to your need. ```bash - cd prometheus-v{version}-linux-amd64.tar.gz + cd prometheus-v{version}-linux-{arch} tar -zcvf ../prometheus-v{new-version}.tar.gz ./ ``` @@ -50,7 +59,7 @@ Download a new installation package from the [Prometheus download page](https:// Execute the following command to upgrade Prometheus: ```bash -tiup cluster patch prometheus-v{new-version}.tar.gz -R prometheus +tiup cluster patch prometheus-v{new-version}.tar.gz -R prometheus --overwrite ``` After the upgrade, you can go to the home page of the Prometheus server (usually at `http://:9090`), click **Status** in the top navigation menu, and then open the **Runtime & Build Information** page to check the Prometheus version and confirm whether the upgrade is successful. @@ -68,20 +77,29 @@ In the following upgrade steps, you need to download the Grafana installation pa ### Step 2. Download the Grafana installation package provided by TiDB -1. Download the TiDB **Server Package** package from the [TiDB download page](https://www.pingcap.com/download) and extract it. -2. In the extracted files, locate `grafana-v{version}-linux-amd64.tar.gz` and extract it. +1. Download the TiDB server package and extract it. Note that your downloading means you agree to the [Privacy Policy](https://www.pingcap.com/privacy-policy/). + + ``` + https://download.pingcap.com/tidb-community-server-{version}-linux-{arch}.tar.gz + ``` + + > **Tip:** + > + > `{version}` in the link indicates the version number of TiDB and `{arch}` indicates the architecture of the system, which can be `amd64` or `arm64`. For example, the download link for `v{{{ .tidb-version }}}` in the `amd64` architecture is `https://download.pingcap.com/tidb-community-toolkit-v{{{ .tidb-version }}}-linux-amd64.tar.gz`. + +2. In the extracted files, locate `grafana-v{version}-linux-{arch}.tar.gz` and extract it. ```bash - tar -xzf grafana-v{version}-linux-amd64.tar.gz + tar -xzf grafana-v{version}-linux-{arch}.tar.gz ``` ### Step 3. Create a new Grafana package that TiUP can use -1. Copy the files extracted in [Step 1](#step-1-download-a-new-grafana-installation-package-from-the-grafana-website), and then use the copied files to replace the files in the `./grafana-v{version}-linux-amd64/` directory extracted in [Step 2](#step-2-download-the-grafana-installation-package-provided-by-tidb). -2. Recompress the `./grafana-v{version}-linux-amd64` directory and name the new compressed package as `grafana-v{new-version}.tar.gz`, where `{new-version}` can be specified according to your need. +1. Copy the files extracted in [Step 1](#step-1-download-a-new-grafana-installation-package-from-the-grafana-website), and then use the copied files to replace the files in the `./grafana-v{version}-linux-{arch}/` directory extracted in [Step 2](#step-2-download-the-grafana-installation-package-provided-by-tidb). +2. Recompress the `./grafana-v{version}-linux-{arch}` directory and name the new compressed package as `grafana-v{new-version}.tar.gz`, where `{new-version}` can be specified according to your need. ```bash - cd grafana-v{version}-linux-amd64.tar.gz + cd grafana-v{version}-linux-{arch} tar -zcvf ../grafana-v{new-version}.tar.gz ./ ``` @@ -90,7 +108,7 @@ In the following upgrade steps, you need to download the Grafana installation pa Execute the following command to upgrade Grafana: ```bash -tiup cluster patch grafana-v{new-version}.tar.gz -R grafana +tiup cluster patch grafana-v{new-version}.tar.gz -R grafana --overwrite ``` @@ -109,7 +127,7 @@ Download the `alertmanager` installation package from the [Prometheus download p Execute the following command to upgrade Alertmanager: ```bash -tiup cluster patch alertmanager-v{new-version}-linux-amd64.tar.gz -R alertmanager +tiup cluster patch alertmanager-v{new-version}-linux-{arch}.tar.gz -R alertmanager --overwrite ``` After the upgrade, you can go to the home page of the Alertmanager server (usually at `http://:9093`), click **Status** in the top navigation menu, and then check the Alertmanager version to confirm whether the upgrade is successful. \ No newline at end of file diff --git a/upgrade-tidb-using-tiup.md b/upgrade-tidb-using-tiup.md index 3f3fda8b386bf..0f34a444f54db 100644 --- a/upgrade-tidb-using-tiup.md +++ b/upgrade-tidb-using-tiup.md @@ -1,43 +1,43 @@ --- title: Upgrade TiDB Using TiUP summary: Learn how to upgrade TiDB using TiUP. -aliases: ['/docs/dev/upgrade-tidb-using-tiup/','/docs/dev/how-to/upgrade/using-tiup/','/tidb/dev/upgrade-tidb-using-tiup-offline','/docs/dev/upgrade-tidb-using-tiup-offline/'] --- # Upgrade TiDB Using TiUP This document is targeted for the following upgrade paths: -- Upgrade from TiDB 4.0 versions to TiDB 7.4. -- Upgrade from TiDB 5.0-5.4 versions to TiDB 7.4. -- Upgrade from TiDB 6.0-6.6 to TiDB 7.4. -- Upgrade from TiDB 7.0-7.3 to TiDB 7.4. +- Upgrade from TiDB 4.0 versions to TiDB 7.5. +- Upgrade from TiDB 5.0-5.4 versions to TiDB 7.5. +- Upgrade from TiDB 6.0-6.6 to TiDB 7.5. +- Upgrade from TiDB 7.0-7.4 to TiDB 7.5. > **Warning:** > > 1. You cannot upgrade TiFlash online from versions earlier than 5.3 to 5.3 or later. Instead, you must first stop all the TiFlash instances of the early version, and then upgrade the cluster offline. If other components (such as TiDB and TiKV) do not support an online upgrade, follow the instructions in warnings in [Online upgrade](#online-upgrade). > 2. **DO NOT** run DDL statements during the upgrade process. Otherwise, the issue of undefined behavior might occur. > 3. **DO NOT** upgrade a TiDB cluster when a DDL statement is being executed in the cluster (usually for the time-consuming DDL statements such as `ADD INDEX` and the column type changes). Before the upgrade, it is recommended to use the [`ADMIN SHOW DDL`](/sql-statements/sql-statement-admin-show-ddl.md) command to check whether the TiDB cluster has an ongoing DDL job. If the cluster has a DDL job, to upgrade the cluster, wait until the DDL execution is finished or use the [`ADMIN CANCEL DDL`](/sql-statements/sql-statement-admin-cancel-ddl.md) command to cancel the DDL job before you upgrade the cluster. -> -> If the TiDB version before upgrade is v7.1.0 or later, you can ignore the preceding warnings 2 and 3. For more information, see [TiDB Smooth Upgrade](/smooth-upgrade-tidb.md). +> 4. If the TiDB version before upgrade is 7.1.0 or later, you can ignore the preceding warnings 2 and 3. For more information, see [limitations on using TiDB smooth upgrade](/smooth-upgrade-tidb.md#limitations). +> 5. Be sure to read [limitations on user operations](/smooth-upgrade-tidb.md#limitations-on-user-operations) before upgrading your TiDB cluster using TiUP. > **Note:** > -> - If your cluster to be upgraded is v3.1 or an earlier version (v3.0 or v2.1), the direct upgrade to v7.4.0 is not supported. You need to upgrade your cluster first to v4.0 and then to v7.4.0. +> - If your cluster to be upgraded is v3.1 or an earlier version (v3.0 or v2.1), the direct upgrade to v7.5.0 or a later v7.5.x version is not supported. You need to upgrade your cluster first to v4.0 and then to the target TiDB version. +> - If your cluster to be upgraded is earlier than v6.2, the upgrade might get stuck when you upgrade the cluster to v6.2 or later versions in some scenarios. You can refer to [How to fix the issue](#how-to-fix-the-issue-that-the-upgrade-gets-stuck-when-upgrading-to-v620-or-later-versions). > - TiDB nodes use the value of the [`server-version`](/tidb-configuration-file.md#server-version) configuration item to verify the current TiDB version. Therefore, to avoid unexpected behaviors, before upgrading the TiDB cluster, you need to set the value of `server-version` to empty or the real version of the current TiDB cluster. ## Upgrade caveat - TiDB currently does not support version downgrade or rolling back to an earlier version after the upgrade. -- For the v4.0 cluster managed using TiDB Ansible, you need to import the cluster to TiUP (`tiup cluster`) for new management according to [Upgrade TiDB Using TiUP (v4.0)](https://docs.pingcap.com/tidb/v4.0/upgrade-tidb-using-tiup#import-tidb-ansible-and-the-inventoryini-configuration-to-tiup). Then you can upgrade the cluster to v7.4.0 according to this document. -- To update versions earlier than v3.0 to v7.4.0: - 1. Update this version to 3.0 using [TiDB Ansible](https://docs.pingcap.com/tidb/v3.0/upgrade-tidb-using-ansible). +- For the v4.0 cluster managed using TiDB Ansible, you need to import the cluster to TiUP (`tiup cluster`) for new management according to [Upgrade TiDB Using TiUP (v4.0)](https://docs-archive.pingcap.com/tidb/v4.0/upgrade-tidb-using-tiup#import-tidb-ansible-and-the-inventoryini-configuration-to-tiup). Then you can upgrade the cluster to v{{{ .tidb-version }}} according to this document. +- To update versions earlier than v3.0 to v{{{ .tidb-version }}}: + 1. Update this version to 3.0 using [TiDB Ansible](https://docs-archive.pingcap.com/tidb/v3.0/upgrade-tidb-using-ansible). 2. Use TiUP (`tiup cluster`) to import the TiDB Ansible configuration. - 3. Update the 3.0 version to 4.0 according to [Upgrade TiDB Using TiUP (v4.0)](https://docs.pingcap.com/tidb/v4.0/upgrade-tidb-using-tiup#import-tidb-ansible-and-the-inventoryini-configuration-to-tiup). - 4. Upgrade the cluster to v7.4.0 according to this document. + 3. Update the 3.0 version to 4.0 according to [Upgrade TiDB Using TiUP (v4.0)](https://docs-archive.pingcap.com/tidb/v4.0/upgrade-tidb-using-tiup#import-tidb-ansible-and-the-inventoryini-configuration-to-tiup). + 4. Upgrade the cluster to v{{{ .tidb-version }}} according to this document. - Support upgrading the versions of TiDB Binlog, TiCDC, TiFlash, and other components. - When upgrading TiFlash from versions earlier than v6.3.0 to v6.3.0 and later versions, note that the CPU must support the AVX2 instruction set under the Linux AMD64 architecture and the ARMv8 instruction set architecture under the Linux ARM64 architecture. For details, see the description in [v6.3.0 Release Notes](/releases/release-6.3.0.md#others). -- For detailed compatibility changes of different versions, see the [Release Notes](/releases/release-notes.md) of each version. Modify your cluster configuration according to the "Compatibility Changes" section of the corresponding release notes. +- For detailed compatibility changes of different versions, see the [Release Notes](https://docs.pingcap.com/releases/tidb-self-managed/) of each version. Modify your cluster configuration according to the "Compatibility Changes" section of the corresponding release notes. - For clusters that upgrade from versions earlier than v5.3 to v5.3 or later versions, the default deployed Prometheus will upgrade from v2.8.1 to v2.27.1. Prometheus v2.27.1 provides more features and fixes a security issue. Compared with v2.8.1, alert time representation in v2.27.1 is changed. For more details, see [Prometheus commit](https://github.com/prometheus/prometheus/commit/7646cbca328278585be15fa615e22f2a50b47d06) for more details. ## Preparations @@ -46,7 +46,19 @@ This section introduces the preparation works needed before upgrading your TiDB ### Step 1: Review compatibility changes -Review [the compatibility changes](/releases/release-7.4.0.md#compatibility-changes) in TiDB v7.4.0 release notes. If any changes affect your upgrade, take actions accordingly. +Review compatibility changes in TiDB release notes. If any changes affect your upgrade, take actions accordingly. + +The following provides compatibility changes you need to know when you upgrade from v7.5.0 to the current version (v{{{ .tidb-version }}}). If you are upgrading from v7.4.0 or earlier versions to the current version, you might also need to check the compatibility changes introduced in intermediate versions in the corresponding [release notes](https://docs.pingcap.com/releases/tidb-self-managed/). + +- TiDB v7.5.0 [compatibility changes](/releases/release-7.5.0.md#compatibility-changes) +- TiDB v7.5.1 [compatibility changes](/releases/release-7.5.1.md#compatibility-changes) +- TiDB v7.5.2 [compatibility changes](/releases/release-7.5.2.md#compatibility-changes) +- TiDB v7.5.3 [compatibility changes](/releases/release-7.5.3.md#compatibility-changes) +- TiDB v7.5.4 [compatibility changes](/releases/release-7.5.4.md#compatibility-changes) +- TiDB v7.5.5 [compatibility changes](/releases/release-7.5.5.md#compatibility-changes) +- TiDB v7.5.6 [compatibility changes](/releases/release-7.5.6.md#compatibility-changes) +- TiDB v7.5.7 [compatibility changes](/releases/release-7.5.7.md#compatibility-changes) +- [TiDB v7.5.8 Release Notes](https://docs.pingcap.com/tidb/stable/release-7.5.8/) ### Step 2: Upgrade TiUP or TiUP offline mirror @@ -121,7 +133,7 @@ Now, the offline mirror has been upgraded successfully. If an error occurs durin > Skip this step if one of the following situations applies: > > + You have not modified the configuration parameters of the original cluster. Or you have modified the configuration parameters using `tiup cluster` but no more modification is needed. -> + After the upgrade, you want to use v7.4.0's default parameter values for the unmodified configuration items. +> + After the upgrade, you want to use the default parameter values of v{{{ .tidb-version }}} for the unmodified configuration items. 1. Enter the `vi` editing mode to edit the topology file: @@ -137,9 +149,20 @@ Now, the offline mirror has been upgraded successfully. If an error occurs durin > **Note:** > -> Before you upgrade the cluster to v6.6.0, make sure that the parameters you have modified in v4.0 are compatible in v7.4.0. For details, see [TiKV Configuration File](/tikv-configuration-file.md). +> Before you upgrade the cluster to v{{{ .tidb-version }}}, make sure that the parameters you have modified in v4.0 are compatible in v{{{ .tidb-version }}}. For details, see [TiKV Configuration File](/tikv-configuration-file.md). -### Step 4: Check the health status of the current cluster +### Step 4: Check the DDL and backup status of the cluster + +To avoid undefined behaviors or other unexpected problems during the upgrade, it is recommended to check the following items before the upgrade. + +- Cluster DDLs: + + - If you use [smooth upgrade](/smooth-upgrade-tidb.md), you do not need to check the DDL operations of your TiDB cluster. You do not need to wait for the completion of DDL jobs or cancel ongoing DDL jobs. + - If you do not use [smooth upgrade](/smooth-upgrade-tidb.md), it is recommended to use the [`ADMIN SHOW DDL`](/sql-statements/sql-statement-admin-show-ddl.md) statement to check whether ongoing DDL jobs exist. If an ongoing DDL job exists, wait for the completion of its execution or cancel it using the [`ADMIN CANCEL DDL`](/sql-statements/sql-statement-admin-cancel-ddl.md) statement before performing an upgrade. + +- Cluster backup: It is recommended to execute the [`SHOW [BACKUPS|RESTORES]`](/sql-statements/sql-statement-show-backups.md) statement to check whether there is an ongoing backup or restore task in the cluster. If yes, wait for its completion before performing an upgrade. + +### Step 5: Check the health status of the current cluster To avoid the undefined behaviors or other issues during the upgrade, it is recommended to check the health status of Regions of the current cluster before the upgrade. To do that, you can use the `check` sub-command. @@ -154,13 +177,6 @@ After the command is executed, the "Region status" check result will be output. + If the result is "All Regions are healthy", all Regions in the current cluster are healthy and you can continue the upgrade. + If the result is "Regions are not fully healthy: m miss-peer, n pending-peer" with the "Please fix unhealthy regions before other operations." prompt, some Regions in the current cluster are abnormal. You need to troubleshoot the anomalies until the check result becomes "All Regions are healthy". Then you can continue the upgrade. -### Step 5: Check the DDL and backup status of the cluster - -To avoid undefined behaviors or other unexpected problems during the upgrade, it is recommended to check the following items before the upgrade. - -- Cluster DDLs: It is recommended to execute the [`ADMIN SHOW DDL`](/sql-statements/sql-statement-admin-show-ddl.md) statement to check whether there is an ongoing DDL job. If yes, wait for its execution or cancel it by executing the [`ADMIN CANCEL DDL`](/sql-statements/sql-statement-admin-cancel-ddl.md) statement before performing an upgrade. -- Cluster backup: It is recommended to execute the [`SHOW [BACKUPS|RESTORES]`](/sql-statements/sql-statement-show-backups.md) statement to check whether there is an ongoing backup or restore task in the cluster. If yes, wait for its completion before performing an upgrade. - ## Upgrade the TiDB cluster This section describes how to upgrade the TiDB cluster and verify the version after the upgrade. @@ -181,12 +197,12 @@ If your application has a maintenance window for the database to be stopped for tiup cluster upgrade ``` -For example, if you want to upgrade the cluster to v7.4.0: +For example, if you want to upgrade the cluster to v{{{ .tidb-version }}}: {{< copyable "shell-regular" >}} ```shell -tiup cluster upgrade v7.4.0 +tiup cluster upgrade v{{{ .tidb-version }}} ``` > **Note:** @@ -197,13 +213,33 @@ tiup cluster upgrade v7.4.0 > > + To keep a stable performance, make sure that all leaders in a TiKV instance are evicted before stopping the instance. You can set `--transfer-timeout` to a larger value, for example, `--transfer-timeout 3600` (unit: second). > -> + To upgrade TiFlash from versions earlier than 5.3 to 5.3 or later, you should stop TiFlash and then upgrade it. The following steps help you upgrade TiFlash without interrupting other components: -> 1. Stop the TiFlash instance: `tiup cluster stop -R tiflash` -> 2. Upgrade the TiDB cluster without restarting it (only updating the files): `tiup cluster upgrade --offline`, such as `tiup cluster upgrade v6.3.0 --offline` -> 3. Reload the TiDB cluster: `tiup cluster reload `. After the reload, the TiFlash instance is started and you do not need to manually start it. +> + To upgrade TiFlash from versions earlier than v5.3.0 to v5.3.0 or later, you must stop TiFlash and then upgrade it, and the TiUP version must be earlier than v1.12.0. For more information, see [Upgrade TiFlash using TiUP](/tiflash-upgrade-guide.md#upgrade-tiflash-using-tiup). > > + Try to avoid creating a new clustered index table when you apply rolling updates to the clusters using TiDB Binlog. +#### Specify the component version during upgrade + +Starting from tiup-cluster v1.14.0, you can specify certain components to a specific version during cluster upgrade. These components will remain at their fixed version in the subsequent upgrade unless you specify a different version. + +> **Note:** +> +> For components that share a version number, such as TiDB, TiKV, PD, and TiCDC, there are no complete tests to ensure that they work properly in a mixed-version deployment scenario. Ensure that you use this section only in test environments, or with the help of [technical support](/support.md). + +```shell +tiup cluster upgrade -h | grep "version string" + --alertmanager-version string Fix the version of alertmanager and no longer follows the cluster version. + --blackbox-exporter-version string Fix the version of blackbox-exporter and no longer follows the cluster version. + --cdc-version string Fix the version of cdc and no longer follows the cluster version. + --ignore-version-check Ignore checking if target version is bigger than current version. + --node-exporter-version string Fix the version of node-exporter and no longer follows the cluster version. + --pd-version string Fix the version of pd and no longer follows the cluster version. + --tidb-dashboard-version string Fix the version of tidb-dashboard and no longer follows the cluster version. + --tiflash-version string Fix the version of tiflash and no longer follows the cluster version. + --tikv-cdc-version string Fix the version of tikv-cdc and no longer follows the cluster version. + --tikv-version string Fix the version of tikv and no longer follows the cluster version. + --tiproxy-version string Fix the version of tiproxy and no longer follows the cluster version. +``` + #### Offline upgrade 1. Before the offline upgrade, you first need to stop the entire cluster. @@ -214,7 +250,7 @@ tiup cluster upgrade v7.4.0 tiup cluster stop ``` -2. Use the `upgrade` command with the `--offline` option to perform the offline upgrade. Fill in the name of your cluster for `` and the version to upgrade to for ``, such as `v7.4.0`. +2. Use the `upgrade` command with the `--offline` option to perform the offline upgrade. Fill in the name of your cluster for `` and the version to upgrade to for ``, such as `v{{{ .tidb-version }}}`. {{< copyable "shell-regular" >}} @@ -243,7 +279,7 @@ tiup cluster display ``` Cluster type: tidb Cluster name: -Cluster version: v7.4.0 +Cluster version: v{{{ .tidb-version }}} ``` ## FAQ @@ -272,9 +308,33 @@ Re-execute the `tiup cluster upgrade` command to resume the upgrade. The upgrade tiup cluster replay ``` +### How to fix the issue that the upgrade gets stuck when upgrading to v6.2.0 or later versions? + +Starting from v6.2.0, TiDB enables the [concurrent DDL framework](/best-practices/ddl-introduction.md#how-the-online-ddl-asynchronous-change-works-in-tidb) by default to execute concurrent DDLs. This framework changes the DDL job storage from a KV queue to a table queue. This change might cause the upgrade to get stuck in some scenarios. The following are some scenarios that might trigger this issue and the corresponding solutions: + +- Upgrade gets stuck due to plugin loading + + During the upgrade, loading certain plugins that require executing DDL statements might cause the upgrade to get stuck. + + **Solution**: avoid loading plugins during the upgrade. Instead, load plugins only after the upgrade is completed. + +- Upgrade gets stuck due to using the `kill -9` command for offline upgrade + + - Precautions: avoid using the `kill -9` command to perform the offline upgrade. If it is necessary, restart the new version TiDB node after 2 minutes. + - If the upgrade is already stuck, restart the affected TiDB node. If the issue has just occurred, it is recommended to restart the node after 2 minutes. + +- Upgrade gets stuck due to DDL Owner change + + In multi-instance scenarios, network or hardware failures might cause DDL Owner change. If there are unfinished DDL statements in the upgrade phase, the upgrade might get stuck. + + **Solution**: + + 1. Terminate the stuck TiDB node (avoid using `kill -9`). + 2. Restart the new version TiDB node. + ### The evict leader has waited too long during the upgrade. How to skip this step for a quick upgrade? -You can specify `--force`. Then the processes of transferring PD leader and evicting TiKV leader are skipped during the upgrade. The cluster is directly restarted to update the version, which has a great impact on the cluster that runs online. In the following command, `` is the version to upgrade to, such as `v7.4.0`. +You can specify `--force`. Then the processes of transferring PD leader and evicting TiKV leader are skipped during the upgrade. The cluster is directly restarted to update the version, which has a great impact on the cluster that runs online. In the following command, `` is the version to upgrade to, such as `v{{{ .tidb-version }}}`. {{< copyable "shell-regular" >}} @@ -289,5 +349,5 @@ You can upgrade the tool version by using TiUP to install the `ctl` component of {{< copyable "shell-regular" >}} ```shell -tiup install ctl:v7.4.0 +tiup install ctl:v{{{ .tidb-version }}} ``` diff --git a/user-account-management.md b/user-account-management.md index c53352651b347..92b64f453e2a1 100644 --- a/user-account-management.md +++ b/user-account-management.md @@ -1,7 +1,6 @@ --- title: TiDB User Account Management summary: Learn how to manage a TiDB user account. -aliases: ['/docs/dev/user-account-management/','/docs/dev/reference/security/user-account-management/'] --- # TiDB User Account Management @@ -159,7 +158,7 @@ TiDB creates the `'root'@'%'` default account during the database initialization ## Set account resource limits -Currently, TiDB does not support setting account resource limits. +TiDB can limit the resources consumed by users using resource groups. For more information, see [Use resource control to achieve resource isolation](/tidb-resource-control.md). ## Assign account passwords diff --git a/user-defined-variables.md b/user-defined-variables.md index 5a18349c13dd7..6faac81c8cc50 100644 --- a/user-defined-variables.md +++ b/user-defined-variables.md @@ -1,7 +1,6 @@ --- title: User-Defined Variables summary: Learn how to use user-defined variables. -aliases: ['/docs/dev/user-defined-variables/','/docs/dev/reference/sql/language-structure/user-defined-variables/'] --- # User-Defined Variables diff --git a/variables.json b/variables.json new file mode 100644 index 0000000000000..5c8bb9fada220 --- /dev/null +++ b/variables.json @@ -0,0 +1,10 @@ +{ + "tidb": "TiDB", + "tidb-version": "7.5.8", + "tidb-release-date": "2026-09-17", + "self-managed": "TiDB Self-Managed", + "starter": "TiDB Cloud Starter", + "essential": "TiDB Cloud Essential", + "dedicated": "TiDB Cloud Dedicated", + "console-url": "tidbcloud.com" +} diff --git a/views.md b/views.md index 2229ab7e2f275..edc5b783f51f3 100644 --- a/views.md +++ b/views.md @@ -1,7 +1,6 @@ --- title: Views summary: Learn how to use views in TiDB. -aliases: ['/docs/dev/views/','/docs/dev/reference/sql/views/'] --- # Views