From eb323bb34389eb7481e2fde162c2a29e72c678bc Mon Sep 17 00:00:00 2001 From: Manohar Reddy Date: Fri, 12 Jun 2026 10:12:44 +0200 Subject: [PATCH] fix bootstrap script for correct hugePages claims --- bare-metal/bootstrap-cluster.sh | 1 + bare-metal/bootstrap-k3s.sh | 12 +- bare-metal/calc_hugepages.py | 232 ++++++++++++++++++++++++++++++++ 3 files changed, 236 insertions(+), 9 deletions(-) create mode 100644 bare-metal/calc_hugepages.py diff --git a/bare-metal/bootstrap-cluster.sh b/bare-metal/bootstrap-cluster.sh index be40f0bd..90793f39 100755 --- a/bare-metal/bootstrap-cluster.sh +++ b/bare-metal/bootstrap-cluster.sh @@ -593,6 +593,7 @@ main() { done for node_ip in ${storage_private_ips}; do install_sbcli_on_node "$node_ip" "$configure_cmd" & done + wait for node_ip in ${mnodes[@]}; do install_sbcli_on_node "$node_ip" ""; done bootstrap_cluster "${mnodes[0]}" diff --git a/bare-metal/bootstrap-k3s.sh b/bare-metal/bootstrap-k3s.sh index 906c9537..4187a4dc 100755 --- a/bare-metal/bootstrap-k3s.sh +++ b/bare-metal/bootstrap-k3s.sh @@ -1,6 +1,7 @@ #!/bin/bash KEY="$HOME/.ssh/simplyblock-us-east-2.pem" +NR_HUGEPAGES=${NR_HUGEPAGES:-} print_help() { echo "Usage: $0 [options]" @@ -91,9 +92,6 @@ ssh -i $KEY -o StrictHostKeyChecking=no \ sudo yum install -y fio nvme-cli bc; sudo modprobe nvme-tcp sudo modprobe nbd -total_memory_kb=\$(grep MemTotal /proc/meminfo | awk '{print \$2}') -total_memory_mb=\$((total_memory_kb / 1024)) -#hugepages=\$(echo \"\$total_memory_mb * 0.3 / 1\" | bc) hugepages=0 sudo sysctl -w vm.nr_hugepages=\$hugepages @@ -124,9 +122,6 @@ for ((i=1; i<${#mnodes[@]}; i++)); do sudo yum install -y fio nvme-cli bc; sudo modprobe nvme-tcp sudo modprobe nbd - total_memory_kb=\$(grep MemTotal /proc/meminfo | awk '{print \$2}') - total_memory_mb=\$((total_memory_kb / 1024)) - #hugepages=\$(echo \"\$total_memory_mb * 0.3 / 1\" | bc) hugepages=0 sudo sysctl -w vm.nr_hugepages=\$hugepages @@ -161,9 +156,8 @@ if [ "$K8S_SNODE" == "true" ]; then sudo modprobe nvme-tcp sudo modprobe nbd total_memory_kb=\$(grep MemTotal /proc/meminfo | awk '{print \$2}') - total_memory_mb=\$((total_memory_kb / 1024)) - hugepages=\$(echo \"\$total_memory_mb * 0.3 / 1\" | bc) + hugepages=${NR_HUGEPAGES:-\$((total_memory_mb / 4))} sudo sysctl -w vm.nr_hugepages=\$hugepages sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 @@ -196,7 +190,7 @@ if [ "$K8S_SNODE" == "true" ]; then sudo modprobe nbd total_memory_kb=\$(grep MemTotal /proc/meminfo | awk '{print \$2}') total_memory_mb=\$((total_memory_kb / 1024)) - hugepages=\$(echo \"\$total_memory_mb * 0.3 / 1\" | bc) + hugepages=${NR_HUGEPAGES:-\$((total_memory_mb / 4))} sudo sysctl -w vm.nr_hugepages=\$hugepages sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 diff --git a/bare-metal/calc_hugepages.py b/bare-metal/calc_hugepages.py new file mode 100644 index 00000000..6ec365a3 --- /dev/null +++ b/bare-metal/calc_hugepages.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +""" +Calculate required hugepages for a simplyblock storage node. + +Mirrors the exact formula from sbcli: + calculate_unisolated_cores -> calculate_core_allocations -> + calculate_pool_count -> calculate_minimum_hp_memory -> adjust_hugepages + +Usage: + python3 calc_hugepages.py --cpus 8 --ram-mb 15735 --nvme 4 + python3 calc_hugepages.py --cpus 8 --ram-mb 15735 --nvme 4 --core-percentage 65 --max-lvol 10 + python3 calc_hugepages.py --cpus 32 --ram-mb 65536 --nvme 8 --core-percentage 80 --max-lvol 256 --max-size-gb 20480 +""" + +import argparse +import math +import sys + +# Constants from sbcli/simplyblock_core/constants.py +EXTRA_SMALL_POOL_COUNT = 30000 +EXTRA_LARGE_POOL_COUNT = 10240 +EXTRA_HUGE_PAGE_MEMORY = 3221225472 # 3 GiB + + +def calculate_unisolated_cores(total_cores, cores_percentage=0): + if cores_percentage: + return math.ceil(total_cores * (100 - cores_percentage) / 100) + if total_cores <= 10: + return 2 + if total_cores <= 20: + return 3 + if total_cores <= 28: + return 4 + return math.ceil(total_cores * 0.15) + + +def calculate_core_allocations(isolated_count, alceml_count): + """ + Simplified (non-hyperthreading-aware) version of sbcli's + calculate_core_allocations. Returns (poller_cores, distrib_cores). + """ + remaining = isolated_count + + if isolated_count < 12: + # app + jm + jc_singleton + 1 alceml + lvol_poller = 5 fixed + reserved = min(5, remaining) + remaining -= reserved + elif isolated_count < 22: + # app + jm + jc_singleton + 2 alceml + lvol_poller = 6 fixed + reserved = min(6, remaining) + remaining -= reserved + else: + # app + jm + jc_singleton + lvol_poller + alceml_count + reserved = min(4 + alceml_count, remaining) + remaining -= reserved + + dp = remaining // 2 + if dp >= 17: + distrib = 24 + poller = remaining - 24 + elif dp >= 12: + distrib = 12 + poller = remaining - 12 + else: + distrib = dp + poller = dp + + # leftover core goes to pollers + leftover = remaining - distrib - poller + poller += leftover + + return poller, distrib + + +def calculate_pool_count(alceml_count, number_of_distribs, cpu_count, poller_count): + poller_number = poller_count if poller_count else cpu_count + + small_pool = ( + 384 * (alceml_count + number_of_distribs + 3 + poller_count) + + (6 + alceml_count + number_of_distribs) * poller_number * 127 + + 384 + 128 * poller_number + + EXTRA_SMALL_POOL_COUNT + ) + large_pool = ( + 48 * (alceml_count + number_of_distribs + 3 + poller_count) + + (6 + alceml_count + number_of_distribs) * 32 + + poller_number * 15 + 384 + 16 * poller_number + + EXTRA_LARGE_POOL_COUNT + ) + return int(small_pool), int(large_pool) + + +def calculate_minimum_hp_memory(small_pool, large_pool, max_lvol, max_prov, cpu_count): + pool_kb = (small_pool * 8 + large_pool * 128) / 1024 + mem_bytes = ( + (4 * cpu_count + 1.1 * pool_kb + 22 * max_lvol) * (1024 * 1024) + + EXTRA_HUGE_PAGE_MEMORY + ) + return int(2.0 * mem_bytes) + + +def adjust_hugepages(pages): + remainder = pages % 500 + pages = pages + (500 - remainder) + decimal_val = float(str(pages)[0] + '.' + str(pages)[1]) + add_val = int(decimal_val * 24) + return pages + add_val + + +def calc(total_cpus, total_ram_mb, nvme_devices, core_percentage, max_lvol, max_size_bytes, nodes_per_socket): + # --- core allocation --- + unisolated = calculate_unisolated_cores(total_cpus, core_percentage) + isolated = total_cpus - unisolated + isolated_per_node = isolated // nodes_per_socket + + poller_cores, distrib_cores = calculate_core_allocations(isolated_per_node, nvme_devices) + + cpu_count = isolated_per_node + poller_count = poller_cores if poller_cores else cpu_count + + # number_of_distribs: default 2, becomes distrib_cores count if > 2 + number_of_distribs = max(2, distrib_cores) + # passed to calculate_pool_count as 2 * number_of_distribs + distribs_arg = 2 * number_of_distribs + + alceml_count = nvme_devices // nodes_per_socket + + # --- pool counts --- + small_pool, large_pool = calculate_pool_count(alceml_count, distribs_arg, cpu_count, poller_count) + + # --- minimum hugepage memory --- + min_hp = calculate_minimum_hp_memory(small_pool, large_pool, max_lvol, max_size_bytes, cpu_count) + spdk_mem = max(min_hp, max_size_bytes) + + # --- K8s pod resource request (MEM_MEGA in kubernetes.py) --- + spdk_mib = math.ceil(spdk_mem / (1024 * 1024)) + mem_mega = (spdk_mib // 2) * 2 + 512 # round to even MiB + 512 MiB buffer + + # --- kernel hugepage count (apply_config in kubernetes.py) --- + # user_baseline=0 (fresh node, baseline file pre-seeded with 0) + raw_pages = (spdk_mem + 1_000_000_000) // 2_000_000 + nr_hugepages = adjust_hugepages(raw_pages) + + hugepage_mem_mib = nr_hugepages * 2 + remaining_mib = total_ram_mb - hugepage_mem_mib + + return { + "isolated_cores": isolated_per_node, + "poller_cores": poller_cores, + "distrib_cores": distrib_cores, + "alceml_count": alceml_count, + "number_of_distribs": number_of_distribs, + "small_pool": small_pool, + "large_pool": large_pool, + "spdk_mem_gib": spdk_mem / 1024**3, + "k8s_request_mib": mem_mega, + "nr_hugepages": nr_hugepages, + "hugepage_mem_mib": hugepage_mem_mib, + "remaining_mib": remaining_mib, + } + + +def main(): + parser = argparse.ArgumentParser( + description="Calculate required hugepages for a simplyblock storage node.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument("--cpus", type=int, required=True, help="Total CPU cores on the node") + parser.add_argument("--ram-mb", type=int, required=True, help="Total RAM in MiB (grep MemTotal /proc/meminfo / 1024)") + parser.add_argument("--nvme", type=int, required=True, help="Number of NVMe devices on the node") + parser.add_argument("--core-percentage", type=int, default=100, help="spec.corePercentage (default: 100)") + parser.add_argument("--max-lvol", type=int, default=256, help="spec.maxLogicalVolumeCount (default: 256)") + parser.add_argument("--max-size-gb", type=int, default=0, help="spec.maxSize in GiB (default: 0)") + parser.add_argument("--nodes-per-socket", type=int, default=1, choices=[1, 2], help="spec.nodesPerSocket (default: 1)") + + args = parser.parse_args() + + max_size_bytes = args.max_size_gb * 1024**3 + + r = calc( + total_cpus=args.cpus, + total_ram_mb=args.ram_mb, + nvme_devices=args.nvme, + core_percentage=args.core_percentage, + max_lvol=args.max_lvol, + max_size_bytes=max_size_bytes, + nodes_per_socket=args.nodes_per_socket, + ) + + print() + print("=== Input ===") + print(f" CPUs: {args.cpus}") + print(f" RAM: {args.ram_mb} MiB ({args.ram_mb/1024:.1f} GiB)") + print(f" NVMe devices: {args.nvme}") + print(f" corePercentage: {args.core_percentage}%") + print(f" maxLogicalVolumeCount: {args.max_lvol}") + print(f" maxSize: {args.max_size_gb} GiB") + print(f" nodesPerSocket: {args.nodes_per_socket}") + + print() + print("=== Core allocation ===") + print(f" Isolated cores: {r['isolated_cores']}") + print(f" Poller cores: {r['poller_cores']}") + print(f" Distrib cores: {r['distrib_cores']} -> number_of_distribs={r['number_of_distribs']}") + print(f" NVMe per node: {r['alceml_count']}") + + print() + print("=== Pool counts ===") + print(f" small_pool: {r['small_pool']:,}") + print(f" large_pool: {r['large_pool']:,}") + + print() + print("=== Memory ===") + print(f" SPDK memory: {r['spdk_mem_gib']:.2f} GiB") + print(f" K8s pod request: hugepages-2Mi: {r['k8s_request_mib']}Mi") + print() + print(f" NR_HUGEPAGES = {r['nr_hugepages']} ({r['hugepage_mem_mib']} MiB = {r['hugepage_mem_mib']/1024:.1f} GiB)") + print(f" Remaining for OS = {r['remaining_mib']} MiB ({r['remaining_mib']/1024:.1f} GiB)") + print() + + if r['remaining_mib'] < 1024: + print(" WARNING: less than 1 GiB remaining for OS and other pods.") + elif r['remaining_mib'] < 2048: + print(" NOTE: remaining memory is tight if management pods also run on this node.") + else: + print(" OK: sufficient memory remaining for OS and node-local pods.") + print() + + +if __name__ == "__main__": + main()