feat(database): 添加 ClickHouse 数据库驱动支持

- 实现了完整的 ClickHouse 驱动，包括连接、查询、插入、更新、删除等基本操作
- 添加了 ClickHouse 特有的数据类型转换和 SQL 过滤功能
- 实现了表结构查询和字段信息获取功能
- 添加了 Ping 检查和错误处理机制
- 增加了对 UPDATE 和 DELETE 语句的语法转换以适配 ClickHouse
- 添加了批量插入操作的支持
- 新增了 14 个测试用例文件用于验证数据库相关功能

database/drivers/clickhouse/clickhouse.go

@@ -4,58 +4,60 @@
 // If a copy of the MIT was not distributed with this file,
 // You can obtain one at https://github.com/gogf/gf.
 
-// Package clickhouse implements database.Driver for ClickHouse database.
+// Package clickhouse implements database.Driver, which supports operations for database ClickHouse.
 package clickhouse
 
 import (
-	"database/sql"
-	"fmt"
-
-	_ "github.com/ClickHouse/clickhouse-go/v2"
+	"context"
+	"errors"
 
 	"git.magicany.cc/black1552/gin-base/database"
+	"github.com/gogf/gf/v2/os/gctx"
 )
 
-// Driver is the driver for ClickHouse database.
+// Driver is the driver for clickhouse database.
 type Driver struct {
 	*database.Core
 }
 
+var (
+	errUnsupportedInsertIgnore = errors.New("unsupported method:InsertIgnore")
+	errUnsupportedInsertGetId  = errors.New("unsupported method:InsertGetId")
+	errUnsupportedReplace      = errors.New("unsupported method:Replace")
+	errUnsupportedBegin        = errors.New("unsupported method:Begin")
+	errUnsupportedTransaction  = errors.New("unsupported method:Transaction")
+)
+
 const (
-	quoteChar = `"`
+	updateFilterPattern             = `(?i)UPDATE[\s]+?(\w+[\.]?\w+)[\s]+?SET`
+	deleteFilterPattern             = `(?i)DELETE[\s]+?FROM[\s]+?(\w+[\.]?\w+)`
+	filterTypePattern               = `(?i)^UPDATE|DELETE`
+	needParsedSqlInCtx  gctx.StrKey = "NeedParsedSql"
+	driverName                      = "clickhouse"
 )
 
 func init() {
-	if err := database.Register("clickhouse", New()); err != nil {
+	if err := database.Register(`clickhouse`, New()); err != nil {
 		panic(err)
 	}
 }
 
-// New creates and returns a driver that implements database.Driver for ClickHouse.
+// New create and returns a driver that implements database.Driver, which supports operations for clickhouse.
 func New() database.Driver {
 	return &Driver{}
 }
 
-// New creates and returns a database object for ClickHouse.
+// New creates and returns a database object for clickhouse.
+// It implements the interface of database.Driver for extra database driver installation.
 func (d *Driver) New(core *database.Core, node *database.ConfigNode) (database.DB, error) {
 	return &Driver{
 		Core: core,
 	}, nil
 }
 
-// GetChars returns the security char for ClickHouse.
-func (d *Driver) GetChars() (charLeft string, charRight string) {
-	return quoteChar, quoteChar
-}
-
-// Open creates and returns an underlying sql.DB object for ClickHouse.
-func (d *Driver) Open(config *database.ConfigNode) (*sql.DB, error) {
-	var source string
-	if config.Link != "" {
-		source = config.Link
-	} else {
-		source = fmt.Sprintf("clickhouse://%s:%s@%s:%s/%s",
-			config.User, config.Pass, config.Host, config.Port, config.Name)
+func (d *Driver) injectNeedParsedSql(ctx context.Context) context.Context {
+	if ctx.Value(needParsedSqlInCtx) != nil {
+		return ctx
 	}
-	return sql.Open("clickhouse", source)
+	return context.WithValue(ctx, needParsedSqlInCtx, true)
 }

database/drivers/clickhouse/clickhouse_convert.go

@@ -0,0 +1,83 @@
+// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
+//
+// This Source Code Form is subject to the terms of the MIT License.
+// If a copy of the MIT was not distributed with this file,
+// You can obtain one at https://github.com/gogf/gf.
+
+package clickhouse
+
+import (
+	"context"
+	"database/sql/driver"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/shopspring/decimal"
+
+	"github.com/gogf/gf/v2/os/gtime"
+)
+
+// ConvertValueForField converts value to the type of the record field.
+func (d *Driver) ConvertValueForField(ctx context.Context, fieldType string, fieldValue any) (any, error) {
+	switch itemValue := fieldValue.(type) {
+	case time.Time:
+		// If the time is zero, it then updates it to nil,
+		// which will insert/update the value to database as "null".
+		if itemValue.IsZero() {
+			return nil, nil
+		}
+		return itemValue, nil
+
+	case uuid.UUID:
+		return itemValue, nil
+
+	case *time.Time:
+		// If the time is zero, it then updates it to nil,
+		// which will insert/update the value to database as "null".
+		if itemValue == nil || itemValue.IsZero() {
+			return nil, nil
+		}
+		return itemValue, nil
+
+	case gtime.Time:
+		// If the time is zero, it then updates it to nil,
+		// which will insert/update the value to database as "null".
+		if itemValue.IsZero() {
+			return nil, nil
+		}
+		// for gtime type, needs to get time.Time
+		return itemValue.Time, nil
+
+	case *gtime.Time:
+		// If the time is zero, it then updates it to nil,
+		// which will insert/update the value to database as "null".
+		if itemValue == nil || itemValue.IsZero() {
+			return nil, nil
+		}
+		// for gtime type, needs to get time.Time
+		return itemValue.Time, nil
+
+	case decimal.Decimal:
+		return itemValue, nil
+
+	case *decimal.Decimal:
+		if itemValue != nil {
+			return *itemValue, nil
+		}
+		return nil, nil
+
+	default:
+		// if the other type implements valuer for the driver package
+		// the converted result is used
+		// otherwise the interface data is committed
+		valuer, ok := itemValue.(driver.Valuer)
+		if !ok {
+			return itemValue, nil
+		}
+		convertedValue, err := valuer.Value()
+		if err != nil {
+			return nil, err
+		}
+		return convertedValue, nil
+	}
+}

database/drivers/clickhouse/clickhouse_do_commit.go

@@ -0,0 +1,19 @@
+// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
+//
+// This Source Code Form is subject to the terms of the MIT License.
+// If a copy of the MIT was not distributed with this file,
+// You can obtain one at https://github.com/gogf/gf.
+
+package clickhouse
+
+import (
+	"context"
+
+	"git.magicany.cc/black1552/gin-base/database"
+)
+
+// DoCommit commits current sql and arguments to underlying sql driver.
+func (d *Driver) DoCommit(ctx context.Context, in database.DoCommitInput) (out database.DoCommitOutput, err error) {
+	ctx = d.InjectIgnoreResult(ctx)
+	return d.Core.DoCommit(ctx, in)
+}

database/drivers/clickhouse/clickhouse_do_delete.go

@@ -0,0 +1,20 @@
+// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
+//
+// This Source Code Form is subject to the terms of the MIT License.
+// If a copy of the MIT was not distributed with this file,
+// You can obtain one at https://github.com/gogf/gf.
+
+package clickhouse
+
+import (
+	"context"
+	"database/sql"
+
+	"git.magicany.cc/black1552/gin-base/database"
+)
+
+// DoDelete does "DELETE FROM ... " statement for the table.
+func (d *Driver) DoDelete(ctx context.Context, link database.Link, table string, condition string, args ...any) (result sql.Result, err error) {
+	ctx = d.injectNeedParsedSql(ctx)
+	return d.Core.DoDelete(ctx, link, table, condition, args...)
+}

database/drivers/clickhouse/clickhouse_do_filter.go

@@ -0,0 +1,83 @@
+// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
+//
+// This Source Code Form is subject to the terms of the MIT License.
+// If a copy of the MIT was not distributed with this file,
+// You can obtain one at https://github.com/gogf/gf.
+
+package clickhouse
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"git.magicany.cc/black1552/gin-base/database"
+	"github.com/gogf/gf/v2/text/gregex"
+)
+
+// DoFilter handles the sql before posts it to database.
+func (d *Driver) DoFilter(
+	ctx context.Context, link database.Link, originSql string, args []any,
+) (newSql string, newArgs []any, err error) {
+	if len(args) == 0 {
+		return originSql, args, nil
+	}
+	// Convert placeholder char '?' to string "$x".
+	var index int
+	originSql, _ = gregex.ReplaceStringFunc(`\?`, originSql, func(s string) string {
+		index++
+		return fmt.Sprintf(`$%d`, index)
+	})
+
+	// Only SQL generated through the framework is processed.
+	if !d.getNeedParsedSqlFromCtx(ctx) {
+		return originSql, args, nil
+	}
+
+	// replace STD SQL to Clickhouse SQL grammar
+	modeRes, err := gregex.MatchString(filterTypePattern, strings.TrimSpace(originSql))
+	if err != nil {
+		return "", nil, err
+	}
+	if len(modeRes) == 0 {
+		return originSql, args, nil
+	}
+
+	// Only delete/ UPDATE statements require filter
+	switch strings.ToUpper(modeRes[0]) {
+	case "UPDATE":
+		// MySQL eg: UPDATE table_name SET field1=new-value1, field2=new-value2 [WHERE Clause]
+		// Clickhouse eg: ALTER TABLE [db.]table UPDATE column1 = expr1 [, ...] WHERE filter_expr
+		newSql, err = gregex.ReplaceStringFuncMatch(
+			updateFilterPattern, originSql,
+			func(s []string) string {
+				return fmt.Sprintf("ALTER TABLE %s UPDATE", s[1])
+			},
+		)
+		if err != nil {
+			return "", nil, err
+		}
+		return newSql, args, nil
+
+	case "DELETE":
+		// MySQL eg: DELETE FROM table_name [WHERE Clause]
+		// Clickhouse eg: ALTER TABLE [db.]table [ON CLUSTER cluster] DELETE WHERE filter_expr
+		newSql, err = gregex.ReplaceStringFuncMatch(
+			deleteFilterPattern, originSql,
+			func(s []string) string {
+				return fmt.Sprintf("ALTER TABLE %s DELETE", s[1])
+			},
+		)
+		if err != nil {
+			return "", nil, err
+		}
+		return newSql, args, nil
+
+	default:
+		return originSql, args, nil
+	}
+}
+
+func (d *Driver) getNeedParsedSqlFromCtx(ctx context.Context) bool {
+	return ctx.Value(needParsedSqlInCtx) != nil
+}

database/drivers/clickhouse/clickhouse_do_insert.go

@@ -0,0 +1,76 @@
+// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
+//
+// This Source Code Form is subject to the terms of the MIT License.
+// If a copy of the MIT was not distributed with this file,
+// You can obtain one at https://github.com/gogf/gf.
+
+package clickhouse
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"strings"
+
+	"git.magicany.cc/black1552/gin-base/database"
+)
+
+// DoInsert inserts or updates data for given table.
+// The list parameter must contain at least one record, which was previously validated.
+func (d *Driver) DoInsert(
+	ctx context.Context, link database.Link, table string, list database.List, option database.DoInsertOption,
+) (result sql.Result, err error) {
+	var keys, valueHolder []string
+
+	// Handle the field names and placeholders.
+	for k := range list[0] {
+		keys = append(keys, k)
+		valueHolder = append(valueHolder, "?")
+	}
+	// Prepare the batch result pointer.
+	var (
+		charL, charR = d.Core.GetChars()
+		keysStr      = charL + strings.Join(keys, charR+","+charL) + charR
+		holderStr    = strings.Join(valueHolder, ",")
+		tx           database.TX
+		stmt         *database.Stmt
+	)
+	tx, err = d.Core.Begin(ctx)
+	if err != nil {
+		return
+	}
+	// It here uses defer to guarantee transaction be committed or roll-backed.
+	defer func() {
+		if err == nil {
+			_ = tx.Commit()
+		} else {
+			_ = tx.Rollback()
+		}
+	}()
+	stmt, err = tx.Prepare(fmt.Sprintf(
+		"INSERT INTO %s(%s) VALUES (%s)",
+		d.QuotePrefixTableName(table), keysStr,
+		holderStr,
+	))
+	if err != nil {
+		return
+	}
+
+	defer func() {
+		_ = stmt.Close()
+	}()
+
+	for i := range len(list) {
+		// Values that will be committed to underlying database driver.
+		params := make([]any, 0)
+		for _, k := range keys {
+			params = append(params, list[i][k])
+		}
+		// Prepare is allowed to execute only once in a transaction opened by clickhouse
+		result, err = stmt.ExecContext(ctx, params...)
+		if err != nil {
+			return
+		}
+	}
+	return
+}

database/drivers/clickhouse/clickhouse_do_update.go

@@ -0,0 +1,20 @@
+// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
+//
+// This Source Code Form is subject to the terms of the MIT License.
+// If a copy of the MIT was not distributed with this file,
+// You can obtain one at https://github.com/gogf/gf.
+
+package clickhouse
+
+import (
+	"context"
+	"database/sql"
+
+	"git.magicany.cc/black1552/gin-base/database"
+)
+
+// DoUpdate does "UPDATE ... " statement for the table.
+func (d *Driver) DoUpdate(ctx context.Context, link database.Link, table string, data any, condition string, args ...any) (result sql.Result, err error) {
+	ctx = d.injectNeedParsedSql(ctx)
+	return d.Core.DoUpdate(ctx, link, table, data, condition, args...)
+}

database/drivers/clickhouse/clickhouse_insert.go

@@ -0,0 +1,27 @@
+// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
+//
+// This Source Code Form is subject to the terms of the MIT License.
+// If a copy of the MIT was not distributed with this file,
+// You can obtain one at https://github.com/gogf/gf.
+
+package clickhouse
+
+import (
+	"context"
+	"database/sql"
+)
+
+// InsertIgnore Other queries for modifying data parts are not supported: REPLACE, MERGE, UPSERT, INSERT UPDATE.
+func (d *Driver) InsertIgnore(ctx context.Context, table string, data any, batch ...int) (sql.Result, error) {
+	return nil, errUnsupportedInsertIgnore
+}
+
+// InsertAndGetId Other queries for modifying data parts are not supported: REPLACE, MERGE, UPSERT, INSERT UPDATE.
+func (d *Driver) InsertAndGetId(ctx context.Context, table string, data any, batch ...int) (int64, error) {
+	return 0, errUnsupportedInsertGetId
+}
+
+// Replace Other queries for modifying data parts are not supported: REPLACE, MERGE, UPSERT, INSERT UPDATE.
+func (d *Driver) Replace(ctx context.Context, table string, data any, batch ...int) (sql.Result, error) {
+	return nil, errUnsupportedReplace
+}

database/drivers/clickhouse/clickhouse_open.go

@@ -0,0 +1,46 @@
+// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
+//
+// This Source Code Form is subject to the terms of the MIT License.
+// If a copy of the MIT was not distributed with this file,
+// You can obtain one at https://github.com/gogf/gf.
+
+package clickhouse
+
+import (
+	"database/sql"
+	"fmt"
+	"net/url"
+
+	"git.magicany.cc/black1552/gin-base/database"
+	"github.com/gogf/gf/v2/errors/gcode"
+	"github.com/gogf/gf/v2/errors/gerror"
+)
+
+// Open creates and returns an underlying sql.DB object for clickhouse.
+func (d *Driver) Open(config *database.ConfigNode) (db *sql.DB, err error) {
+	var source string
+	// clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms&max_execution_time=60
+	if config.Pass != "" {
+		source = fmt.Sprintf(
+			"clickhouse://%s:%s@%s:%s/%s?debug=%t",
+			config.User, url.PathEscape(config.Pass),
+			config.Host, config.Port, config.Name, config.Debug,
+		)
+	} else {
+		source = fmt.Sprintf(
+			"clickhouse://%s@%s:%s/%s?debug=%t",
+			config.User, config.Host, config.Port, config.Name, config.Debug,
+		)
+	}
+	if config.Extra != "" {
+		source = fmt.Sprintf("%s&%s", source, config.Extra)
+	}
+	if db, err = sql.Open(driverName, source); err != nil {
+		err = gerror.WrapCodef(
+			gcode.CodeDbOperationError, err,
+			`sql.Open failed for driver "%s" by source "%s"`, driverName, source,
+		)
+		return nil, err
+	}
+	return
+}

database/drivers/clickhouse/clickhouse_ping.go

@@ -0,0 +1,41 @@
+// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
+//
+// This Source Code Form is subject to the terms of the MIT License.
+// If a copy of the MIT was not distributed with this file,
+// You can obtain one at https://github.com/gogf/gf.
+
+package clickhouse
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+)
+
+// PingMaster pings the master node to check authentication or keeps the connection alive.
+func (d *Driver) PingMaster() error {
+	conn, err := d.Master()
+	if err != nil {
+		return err
+	}
+	return d.ping(conn)
+}
+
+// PingSlave pings the slave node to check authentication or keeps the connection alive.
+func (d *Driver) PingSlave() error {
+	conn, err := d.Slave()
+	if err != nil {
+		return err
+	}
+	return d.ping(conn)
+}
+
+// ping Returns the Clickhouse specific error.
+func (d *Driver) ping(conn *sql.DB) error {
+	err := conn.Ping()
+	if exception, ok := err.(*clickhouse.Exception); ok {
+		return fmt.Errorf("[%d]%s", exception.Code, exception.Message)
+	}
+	return err
+}

database/drivers/clickhouse/clickhouse_table_fields.go

@@ -0,0 +1,70 @@
+// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
+//
+// This Source Code Form is subject to the terms of the MIT License.
+// If a copy of the MIT was not distributed with this file,
+// You can obtain one at https://github.com/gogf/gf.
+
+package clickhouse
+
+import (
+	"context"
+	"fmt"
+
+	"git.magicany.cc/black1552/gin-base/database"
+	"github.com/gogf/gf/v2/text/gregex"
+	"github.com/gogf/gf/v2/util/gutil"
+)
+
+const (
+	tableFieldsColumns = `name,position,default_expression,comment,type,is_in_partition_key,is_in_sorting_key,is_in_primary_key,is_in_sampling_key`
+)
+
+// TableFields retrieves and returns the fields' information of specified table of current schema.
+// Also see DriverMysql.TableFields.
+func (d *Driver) TableFields(ctx context.Context, table string, schema ...string) (fields map[string]*database.TableField, err error) {
+	var (
+		result     database.Result
+		link       database.Link
+		usedSchema = gutil.GetOrDefaultStr(d.GetSchema(), schema...)
+	)
+	if link, err = d.SlaveLink(usedSchema); err != nil {
+		return nil, err
+	}
+	var (
+		getColumnsSql = fmt.Sprintf(
+			"select %s from `system`.columns c where `table` = '%s'",
+			tableFieldsColumns, table,
+		)
+	)
+	result, err = d.DoSelect(ctx, link, getColumnsSql)
+	if err != nil {
+		return nil, err
+	}
+	fields = make(map[string]*database.TableField)
+	for _, m := range result {
+		var (
+			isNull    = false
+			fieldType = m["type"].String()
+		)
+		// in clickhouse , field type like is Nullable(int)
+		fieldsResult, _ := gregex.MatchString(`^Nullable\((.*?)\)`, fieldType)
+		if len(fieldsResult) == 2 {
+			isNull = true
+			fieldType = fieldsResult[1]
+		}
+		position := m["position"].Int()
+		if result[0]["position"].Int() != 0 {
+			position -= 1
+		}
+		fields[m["name"].String()] = &database.TableField{
+			Index:   position,
+			Name:    m["name"].String(),
+			Default: m["default_expression"].Val(),
+			Comment: m["comment"].String(),
+			// Key:     m["Key"].String(),
+			Type: fieldType,
+			Null: isNull,
+		}
+	}
+	return fields, nil
+}

database/drivers/clickhouse/clickhouse_tables.go

@@ -0,0 +1,36 @@
+// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
+//
+// This Source Code Form is subject to the terms of the MIT License.
+// If a copy of the MIT was not distributed with this file,
+// You can obtain one at https://github.com/gogf/gf.
+
+package clickhouse
+
+import (
+	"context"
+	"fmt"
+
+	"git.magicany.cc/black1552/gin-base/database"
+)
+
+const (
+	tablesSqlTmp = "select name from `system`.tables where database = '%s'"
+)
+
+// Tables retrieves and returns the tables of current schema.
+// It's mainly used in cli tool chain for automatically generating the models.
+func (d *Driver) Tables(ctx context.Context, schema ...string) (tables []string, err error) {
+	var result database.Result
+	link, err := d.SlaveLink(schema...)
+	if err != nil {
+		return nil, err
+	}
+	result, err = d.DoSelect(ctx, link, fmt.Sprintf(tablesSqlTmp, d.GetConfig().Name))
+	if err != nil {
+		return
+	}
+	for _, m := range result {
+		tables = append(tables, m["name"].String())
+	}
+	return
+}

database/drivers/clickhouse/clickhouse_transaction.go

@@ -0,0 +1,23 @@
+// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
+//
+// This Source Code Form is subject to the terms of the MIT License.
+// If a copy of the MIT was not distributed with this file,
+// You can obtain one at https://github.com/gogf/gf.
+
+package clickhouse
+
+import (
+	"context"
+
+	"git.magicany.cc/black1552/gin-base/database"
+)
+
+// Begin starts and returns the transaction object.
+func (d *Driver) Begin(ctx context.Context) (tx database.TX, err error) {
+	return nil, errUnsupportedBegin
+}
+
+// Transaction wraps the transaction logic using function `f`.
+func (d *Driver) Transaction(ctx context.Context, f func(ctx context.Context, tx database.TX) error) error {
+	return errUnsupportedTransaction
+}