Skip to content

fix: support catalog-qualified benchmark databases #29

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zgxme wants to merge 4 commits into
base: main
Choose a base branch
from
Open

fix: support catalog-qualified benchmark databases #29

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
31d763d
fix: support catalog-qualified benchmark databases
zgxme Jun 30, 2026
32a6542
fix: run benchmark SQL through mysql stdin
zgxme Jun 30, 2026
77bfc9d
fix: always append last query id SQL
zgxme Jun 30, 2026
7aad94a
debug
zgxme Jul 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion benchmark.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,7 @@ run_session() {
echo "Running set session."
local session_content
session_content=$(envsubst < "$session_file")
if ! engine_run_sql "" "$session_content" false; then
if ! engine_run_sql "" "$session_content"; then
die "Setup session failed"
fi
}
Expand Down Expand Up @@ -531,6 +531,7 @@ run_timed_query() {

local start_time
start_time=$(date +%s%3N)
sleep 1
if engine_run_sql "${db}" "$sql_content"; then
local end_time
end_time=$(date +%s%3N)
Expand Down
121 changes: 71 additions & 50 deletions engines/doris_engine.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,28 @@ source "$(dirname "${BASH_SOURCE[0]}")/../lib/jdbc_utils.sh"

BE_HOSTS_ARR=()

doris_qualified_db() {
local db_name="$1"

if [ -n "${catalog:-}" ] && [ -n "$db_name" ] && [[ "$db_name" != *.* ]]; then
printf '%s.%s\n' "$catalog" "$db_name"
else
printf '%s\n' "$db_name"
fi
}

doris_qualified_table() {
local table_name="$1"
local db_name

db_name="$(doris_qualified_db "${db:-}")"
if [[ "$table_name" != *.* ]] && [ -n "$db_name" ]; then
printf '%s.%s\n' "$db_name" "$table_name"
else
printf '%s\n' "$table_name"
fi
}

any_clear_cache_enabled() {
[[ "${clear_file_cache:-false}" == "true" \
|| "${clear_sys_page_cache:-false}" == "true" ]]
Expand Down Expand Up @@ -204,6 +226,8 @@ engine_init() {
# 2. Execute a SQL file using mysql client
engine_run_sql_file() {
local sql_file="$1"
local apply_session="${2:-true}"
local db_name
local error_file=""
local status=0

Expand All @@ -214,20 +238,22 @@ engine_run_sql_file() {

# Set password environment variable for mysql
export MYSQL_PWD="${password:-}"
db_name="${db:-}"
# DDL setup passes apply_session=false and may create/drop the catalog itself,
# so do not select catalog.db before the catalog exists.
if [ "$apply_session" != "false" ]; then
db_name="$(doris_qualified_db "$db_name")"
fi
local args=(-h"$fe_host" -P"$fe_query_port" -u"$user")
[ -n "$db_name" ] && args+=(-D"$db_name")

error_file="$(mktemp "${TMPDIR:-/tmp}/doris_mysql_stderr.XXXXXX")" || {
echo "ERROR: Failed to create temporary stderr file" >&2
return 1
}

# Execute the SQL file
if mysql \
-h"$fe_host" \
-P"$fe_query_port" \
-u"$user" \
-D"$db" \
2>"$error_file" \
< "$sql_file"; then
if mysql "${args[@]}" 2>"$error_file" < "$sql_file"; then
rm -f "$error_file"
return 0
else
Expand All @@ -245,8 +271,8 @@ engine_run_sql_file() {
engine_run_sql() {
local db="$1"
local sql_statement="$2"
local capture_last_query_id="${3:-true}"
local error_file=""
local sql_file=""
local status=0

if [ -z "$sql_statement" ]; then
Expand All @@ -259,37 +285,44 @@ engine_run_sql() {

# Build mysql command arguments
local args=(-h"$fe_host" -P"$fe_query_port" -u"$user")
[ -n "${catalog:-}" ] && [ -n "$db" ] && db="${catalog}.${db}"
db="$(doris_qualified_db "$db")"
[ -n "$db" ] && args+=(-D"$db")

local mysql_sql="$sql_statement"
if [ "$capture_last_query_id" = "true" ]; then
local normalized_sql
normalized_sql=$(printf '%s' "$sql_statement" | sed -e ':trim' -e 's/[[:space:]]*$//' \
-e '/;$/ { s/;*$//; b trim; }')
mysql_sql="${normalized_sql}; select last_query_id();"
fi

local last_query_id_file="${RESULT_DIR:-/tmp}/.last_query_id"
error_file="$(mktemp "${TMPDIR:-/tmp}/doris_mysql_stderr.XXXXXX")" || {
echo "ERROR: Failed to create temporary stderr file" >&2
return 1
}
if output=$(mysql "${args[@]}" --batch --skip-column-names \
-e "$mysql_sql" 2>"$error_file"); then
sql_file="$(mktemp "${TMPDIR:-/tmp}/doris_mysql_sql.XXXXXX")" || {
echo "ERROR: Failed to create temporary SQL file" >&2
rm -f "$error_file"
if [ "$capture_last_query_id" = "true" ]; then
# The last non-empty line of stdout is the query ID.
echo "$output" | tail -n 1 > "$last_query_id_file"
fi
return 1
}
printf '%s\n' "$sql_statement" > "$sql_file"
local sql_tail="$sql_statement"
while :; do
case "$sql_tail" in
*[[:space:]]) sql_tail="${sql_tail%?}" ;;
*) break ;;
esac
done
case "$sql_tail" in
*";") printf 'select last_query_id();\n' >> "$sql_file" ;;
*) printf ';\nselect last_query_id();\n' >> "$sql_file" ;;
esac
if output=$(mysql "${args[@]}" --batch --skip-column-names < "$sql_file" 2>"$error_file"); then
rm -f "$sql_file"
rm -f "$error_file"
# The last non-empty line of stdout is the query ID.
echo "$output" | sed '/^[[:space:]]*$/d' | tail -n 1 > "$last_query_id_file"
return 0
else
status=$?
echo "ERROR: Failed to execute SQL statement: $sql_statement" >&2
if [ -s "$error_file" ]; then
cat "$error_file" >&2
fi
rm -f "$error_file"
rm -f "$error_file" "$sql_file"
return "$status"
fi
}
Expand All @@ -309,9 +342,7 @@ engine_list_tables() {
local db_name="$1"
local args=(-h"$fe_host" -P"$fe_query_port" -u"$user" -N -s)

if [ -n "${catalog:-}" ]; then
db_name="${catalog}.${db_name}"
fi
db_name="$(doris_qualified_db "$db_name")"
[ -n "$db_name" ] && args+=(-D"$db_name")

export MYSQL_PWD="${password:-}"
Expand All @@ -323,9 +354,7 @@ engine_drop_stats() {
local table="$2"
local args=(-h"$fe_host" -P"$fe_query_port" -u"$user")

if [ -n "${catalog:-}" ]; then
db_name="${catalog}.${db_name}"
fi
db_name="$(doris_qualified_db "$db_name")"
[ -n "$db_name" ] && args+=(-D"$db_name")

export MYSQL_PWD="${password:-}"
Expand All @@ -339,9 +368,7 @@ engine_analyze_table() {
local args=(-h"$fe_host" -P"$fe_query_port" -u"$user")
local sql=""

if [ -n "${catalog:-}" ]; then
db_name="${catalog}.${db_name}"
fi
db_name="$(doris_qualified_db "$db_name")"
[ -n "$db_name" ] && args+=(-D"$db_name")

case "${analyze_type}" in
Expand Down Expand Up @@ -369,9 +396,7 @@ engine_show_column_stats() {
local table="$2"
local args=(-h"$fe_host" -P"$fe_query_port" -u"$user")

if [ -n "${catalog:-}" ]; then
db_name="${catalog}.${db_name}"
fi
db_name="$(doris_qualified_db "$db_name")"
[ -n "$db_name" ] && args+=(-D"$db_name")

export MYSQL_PWD="${password:-}"
Expand All @@ -384,18 +409,10 @@ engine_get_table_rows() {
local port="${fe_query_port:-9030}"
local sys_user="${user:-root}"
local current_db="${db:-}"
local qualified_table="$table"
local qualified_table

if [ -n "${catalog:-}" ] && [ -n "$current_db" ]; then
current_db="${catalog}.${current_db}"
fi

# Support bare table names in benchmark.yaml tables config when a catalog
# is configured, while still allowing callers to pass db.table or
# catalog.db.table explicitly.
if [[ "$qualified_table" != *.* ]] && [ -n "${catalog:-}" ] && [ -n "${db:-}" ]; then
qualified_table="${catalog}.${db}.${qualified_table}"
fi
current_db="$(doris_qualified_db "$current_db")"
qualified_table="$(doris_qualified_table "$table")"

# Do not use `export MYSQL_PWD` to avoid environment pollution
MYSQL_PWD="${password:-}" mysql -h"${host}" -P"${port}" -u"${sys_user}" "${current_db}" \
Expand Down Expand Up @@ -688,7 +705,9 @@ run_clear_cache_actions() {
engine_get_jdbc_datasource() {
# Escape any special characters in the password
local escaped_password
local jdbc_db
escaped_password=$(xml_escape "${password:-}")
jdbc_db="$(doris_qualified_db "${db:-}")"

cat << EOF
<JDBCDataSource guiclass="TestBeanGUI" testclass="JDBCDataSource" testname="JDBC Connection Configuration" enabled="true">
Expand All @@ -697,7 +716,7 @@ engine_get_jdbc_datasource() {
<stringProp name="connectionAge">5000</stringProp>
<stringProp name="connectionProperties"></stringProp>
<stringProp name="dataSource">${ENGINE_TYPE:-Doris}</stringProp>
<stringProp name="dbUrl">jdbc:mysql://${fe_host}:${fe_query_port}/${db}</stringProp>
<stringProp name="dbUrl">jdbc:mysql://${fe_host}:${fe_query_port}/${jdbc_db}</stringProp>
<stringProp name="driver">com.mysql.cj.jdbc.Driver</stringProp>
<stringProp name="keepAlive">true</stringProp>
<stringProp name="password">${escaped_password}</stringProp>
Expand Down Expand Up @@ -754,7 +773,7 @@ engine_get_plan() {
local sql_statement="$2"
export MYSQL_PWD="${password:-}"
local args=(-h"$fe_host" -P"$fe_query_port" -u"$user" -N -s)
[ -n "${catalog:-}" ] && db_name="${catalog}.${db_name}"
db_name="$(doris_qualified_db "$db_name")"
[ -n "$db_name" ] && args+=(-D"$db_name")
mysql "${args[@]}" -e "explain memo plan ${sql_statement}" 2>/dev/null || true
}
Expand All @@ -763,7 +782,9 @@ engine_get_plan() {
engine_get_version() {
export MYSQL_PWD="${password:-}"
local args=(-h"$fe_host" -P"$fe_query_port" -u"$user")
[ -n "${db:-}" ] && args+=(-D"$db")
local db_name
db_name="$(doris_qualified_db "${db:-}")"
[ -n "$db_name" ] && args+=(-D"$db_name")

local version
version=$(mysql "${args[@]}" -N -s -e "SHOW VARIABLES LIKE 'version_comment';" 2>/dev/null | cut -f2- || true)
Expand Down
1 change: 0 additions & 1 deletion engines/interface.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ engine_run_sql_file() {
#
# @param $1: Database name
# @param $2: SQL statement to execute
# @param $3: Optional. Whether to apply session SQL first (default: true)
# @return: 0 on success, non-zero on failure
engine_run_sql() {
local db="$1"
Expand Down
Loading