feat: Refactor cacct app to configure TSDB custom queries

.github/workflows/step_tests-lint.yml

@@ -28,5 +28,5 @@ jobs:
       - name: Lint
         uses: golangci/golangci-lint-action@v9
         with:
-          version: v2.11.4
+          version: v2.13.2
           args: --timeout=5m

.golangci.yml

@@ -8,11 +8,13 @@ linters:
     - dupword
     - err113
     - exhaustruct
+    - exhaustruct_v5
     - funlen
     - gochecknoglobals
     - gochecknoinits
     - gocognit
     - gocritic
+    - goconst
     - gocyclo
     - gosmopolitan
     - ireturn
@@ -57,6 +59,9 @@ linters:
       ignore-string-values: 
         - cpu
         - info
+      ignore-tests: true
+      ignore-map-keys: true
+      min: 5
     gomoddirectives:
       replace-allow-list:
         - github.com/grafana/pyroscope/ebpf
@@ -81,8 +86,9 @@ linters:
           - gosec
           - noctx
           - unqueryvet
+          - goconst
         # text: "(G705: XSS via taint analysis|G704: SSRF via taint analysis|G703: Path traversal via taint analysis|G702: Command injection via taint analysis|G306: Expect WriteFile permissions to be 0600 or less|G120: Parsing form data without limiting request body size can allow memory exhaustion (use http.MaxBytesReader)|net/http/httptest.NewRequest must not be called. use net/http/httptest.NewRequestWithContext)"
-        path: scripts/mock_servers
+        path: scripts
       - linters: 
           - exhaustive
           - intrange
@@ -98,6 +104,7 @@ formatters:
     - gofmt
     - gofumpt
     - goimports
+    - swaggo
   exclusions:
     generated: lax
     paths:

Makefile

@@ -54,7 +54,7 @@ PROMU_TEST_CONF ?= .promu/.promu-go-test.yml
 ifeq ($(CGO_APPS), 1)
 	PROMU_CONF ?= .promu/.promu-cgo.yml
 	pkgs := ./pkg/sqlite3 ./pkg/api/cli \
-			./pkg/api/db ./pkg/api/db/migrator ./pkg/api/helper \
+			./pkg/api/db ./pkg/api/db/migrator \
 			./pkg/api/resource ./pkg/api/resource/slurm ./pkg/api/resource/lsf \
 			./pkg/api/resource/openstack ./pkg/api/resource/k8s \
 			./pkg/api/updater ./pkg/api/updater/tsdb \
@@ -193,12 +193,14 @@ test-e2e: $(PROMTOOL) build pkg/collector/testdata/sys/.unpacked pkg/collector/t
 	./scripts/e2e-test.sh -s tool-relabel-configs
 	./scripts/e2e-test.sh -s tool-web-config
 	./scripts/e2e-test.sh -s cacct-default-format
+	./scripts/e2e-test.sh -s cacct-help-format
 	./scripts/e2e-test.sh -s cacct-long-format
 	./scripts/e2e-test.sh -s cacct-custom-format
 	./scripts/e2e-test.sh -s cacct-admin-user
 	./scripts/e2e-test.sh -s cacct-admin-user-all-users
 	./scripts/e2e-test.sh -s cacct-forbid-query
 	./scripts/e2e-test.sh -s cacct-invalid-config
+	./scripts/e2e-test.sh -s cacct-bad-config
 	./scripts/e2e-test.sh -s cacct-tsdata
 	./scripts/e2e-test.sh -s cacct-tsdata-fail
 else
@@ -290,12 +292,14 @@ test-e2e-update: build pkg/collector/testdata/sys/.unpacked pkg/collector/testda
 	./scripts/e2e-test.sh -s tool-relabel-configs -u || true
 	./scripts/e2e-test.sh -s tool-web-config -u || true
 	./scripts/e2e-test.sh -s cacct-default-format -u || true
+	./scripts/e2e-test.sh -s cacct-help-format -u || true
 	./scripts/e2e-test.sh -s cacct-long-format -u || true 
 	./scripts/e2e-test.sh -s cacct-custom-format -u || true
 	./scripts/e2e-test.sh -s cacct-admin-user -u || true
 	./scripts/e2e-test.sh -s cacct-admin-user-all-users -u || true
 	./scripts/e2e-test.sh -s cacct-forbid-query -u || true
 	./scripts/e2e-test.sh -s cacct-invalid-config -u || true
+	./scripts/e2e-test.sh -s cacct-bad-config -u || true
 	./scripts/e2e-test.sh -s cacct-tsdata -u || true
 	./scripts/e2e-test.sh -s cacct-tsdata-fail -u || true
 else

build/config/cacct/cacct.yml

@@ -294,41 +294,96 @@
 #     #
 #     http_headers: {}
 
-#   # To dump the time series data for each metric, this section must be configured.
-#   # The key name is the name of the metric and value is the PromQL query to get
-#   # time series data. The placeholder `%s` will be replaced by list of job IDs delimited
-#   # by `|` which is the syntax expected by TSDB server.
+#   # List of TSDB queries
+#   #
+#   # The keys `name`, `query` and `kind` are mandatory and more over `name` must be
+#   # unique. The key `title` and `help` can be used to provide human friendly name and
+#   # help text. For the moment, only `range` kind is supported.
 #   #
 #   # If the TSDB server has been configured with the recording rules generated by `ceems_tool`
 #   # the following queries should work out-of-the-box.
 #   #
+#   #
+#   # Available template variables:
+#   #
+#   # - UUIDs -> UUIDs string delimited by "|", e.g., 123|345|567
+#   # - ScrapeInterval -> Scrape interval of TSDB in time.Duration format, e.g., 15s, 1m
+#   # - ScrapeIntervalMilli -> Scrape interval of TSDB in milliseconds, e.g., 15000, 60000
+#   # - EvaluationInterval -> Evaluation interval of TSDB in time.Duration format, e.g., 15s, 1m
+#   # - EvaluationIntervalMilli -> Evaluation interval of TSDB in milliseconds, e.g., 15s, 1m
+#   # - RateInterval -> Rate interval in time.Duration format. It is estimated based on Scrape interval as 4*scrape_interval
+#   # - Range -> Duration of interval where aggregation is being made in time.Duration format
+#   #
+#   # IMPORTANT: Always use backticks around {{.UUIDs}} template variable to escape characters
+#   # like "[" and "]" which can be found in identifiers of resource managers like LSF.
+#   #
 #   queries:
-#     # CPU utilisation
-#     cpu_usage: uuid:ceems_cpu_usage:ratio_irate{uuid=~"%s"}
-
-#     # CPU Memory utilisation
-#     cpu_mem_usage: uuid:ceems_cpu_memory_usage:ratio{uuid=~"%s"}
-
-#     # Host power usage in Watts
-#     host_power_usage: uuid:ceems_host_power_watts:pue{uuid=~"%s"}
-
-#     # Host emissions in g/s
-#     host_emissions: uuid:ceems_host_emissions_g_s:pue{uuid=~"%s"}
-
-#     # GPU utilization
-#     avg_gpu_usage: uuid:ceems_gpu_usage:ratio{uuid=~"%s"}
-
-#     # GPU memory utilization
-#     avg_gpu_mem_usage: uuid:ceems_gpu_memory_usage:ratio{uuid=~"%s"}
-
-#     # GPU power usage in Watts
-#     gpu_power_usage: uuid:ceems_gpu_power_watts:pue{uuid=~"%s"}
-
-#     # GPU emissions in g/s
-#     gpu_emissions: uuid:ceems_gpu_emissions_g_s:pue{uuid=~"%s"}
-
-#     # Read IO bytes
-#     io_read_bytes: irate(ceems_ebpf_read_bytes_total{uuid=~"%s"}[1m])
-
-#     # Write IO bytes
-#     io_write_bytes: irate(ceems_ebpf_write_bytes_total{uuid=~"%s"}[1m])
+#    # # CPU utilization
+#    # - name: cpu_usage 
+#    #   title: "CPU Usage (%)"
+#    #   help: "Usage of CPUs in %"
+#    #   query: uuid:ceems_cpu_usage:ratio_irate{uuid=~`{{.UUIDs}}`}
+#    #   kind: range
+#    #    
+#    # # CPU Memory utilization
+#    # - name: cpu_mem_usage
+#    #   title: "CPU Memory Usage (%)" 
+#    #   help: "Ratio of memory used to memory reserved of CPU in %"
+#    #   query: uuid:ceems_cpu_memory_usage:ratio{uuid=~`{{.UUIDs}}`}
+#    #   kind: range
+#    #      
+#    # # Host power usage in Watts
+#    # - name: host_power_usage
+#    #   title: "Host Power usage (W)" 
+#    #   help: "Instanteous power usage of host in Watts"
+#    #   query: uuid:ceems_host_power_watts:pue{uuid=~`{{.UUIDs}}`}
+#    #   kind: range
+#    #
+#    # # Host emissions in g/s
+#    # - name: host_emissions 
+#    #   title: "Host Eq. Emissions Rate (g/s)"
+#    #   help: "Instanteous emissions rate of host in g/s"
+#    #   query: uuid:ceems_host_emissions_g_s:pue{uuid=~`{{.UUIDs}}`}
+#    #   kind: range
+#    #
+#    # # GPU utilization
+#    # - name: avg_gpu_usage
+#    #   title: "GPU Usage (%)" 
+#    #   help: "Usage of GPU in %"
+#    #   query: uuid:ceems_gpu_usage:ratio{uuid=~`{{.UUIDs}}`}
+#    #   kind: range
+#    #
+#    # # GPU memory utilization
+#    # - name: avg_gpu_mem_usage
+#    #   title: "GPU Memory Usage (%)" 
+#    #   help: "Ratio of memory used to memory reserved of GPU in %"
+#    #   query: uuid:ceems_gpu_memory_usage:ratio{uuid=~`{{.UUIDs}}`}
+#    #   kind: range
+#    #
+#    # # GPU power usage in Watts
+#    # - name: gpu_power_usage
+#    #   title: "GPU Power Usage (W)"
+#    #   help: "Instanteous Power Usage of GPU in Watts"
+#    #   query: uuid:ceems_gpu_power_watts:pue{uuid=~`{{.UUIDs}}`}
+#    #   kind: range
+#    #
+#    # # GPU emissions in g/s
+#    # - name: gpu_emissions 
+#    #   title: "GPU Eq. Emission Rate (g/s)"
+#    #   help: "Instanteous emissions rate of GPU in g/s"
+#    #   query: uuid:ceems_gpu_emissions_g_s:pue{uuid=~`{{.UUIDs}}`}
+#    #   kind: range
+#    #
+#    # # Read IO bytes/s
+#    # - name: io_read_bytes 
+#    #   title: "IO Read Bandwidth (b/s)"
+#    #   help: "Instanteous IO read bandwidth in bytes/s"
+#    #   query: irate(ceems_ebpf_read_bytes_total{uuid=~`{{.UUIDs}}`}[1m])
+#    #   kind: range
+#    #
+#    # # Write IO bytes/s
+#    # - name: io_write_bytes 
+#    #   title: "IO Write Bandwidth (b/s)"
+#    #   help: "Instanteous IO write bandwidth in bytes/s"
+#    #   query: irate(ceems_ebpf_write_bytes_total{uuid=~`{{.UUIDs}}`}[1m])
+#    #   kind: range

cmd/cacct/api.go

@@ -10,7 +10,6 @@ import (
 	"log/slog"
 	"net/http"
 	"net/url"
-	"os"
 	"slices"
 	"strconv"
 	"strings"
@@ -35,8 +34,7 @@ func stats(
 	jobs []string,
 	userNames []string,
 	fields []string,
-	tsData bool,
-	tsDataOut string,
+	includeSummaryStats bool,
 ) ([]models.Unit, []models.Usage, error) {
 	// Add user header to HTTP config
 	userHeaders := http_config.Header{
@@ -93,7 +91,7 @@ func stats(
 	// Even if normal user make a request by requesting
 	// with --user flag, if that user is not in admin list, empty
 	// result will be returned
-	var unitsReqURL, usageReqURL string
+	var unitsReqURL, usageReqURL *url.URL
 
 	if len(userNames) > 0 {
 		// If --user flag does not contain special value "all", add them to the query.
@@ -105,14 +103,14 @@ func stats(
 			}
 		}
 
-		unitsReqURL = apiURL.JoinPath("/api/v1/units/admin").String()
-		usageReqURL = apiURL.JoinPath("/api/v1/usage/current/admin").String()
+		unitsReqURL = apiURL.JoinPath("/api/v1/units/admin")
+		usageReqURL = apiURL.JoinPath("/api/v1/usage/current/admin")
 	} else {
-		unitsReqURL = apiURL.JoinPath("/api/v1/units").String()
-		usageReqURL = apiURL.JoinPath("/api/v1/usage/current").String()
+		unitsReqURL = apiURL.JoinPath("/api/v1/units")
+		usageReqURL = apiURL.JoinPath("/api/v1/usage/current")
 	}
 
-	logger.Debug("Request to fetch units", "url", unitsReqURL, "units_query", urlValues.Encode())
+	logger.Debug("Request to fetch units", "url", unitsReqURL.Redacted(), "units_query", urlValues.Encode())
 
 	// If CEEMS URL is available make a API request
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
@@ -126,51 +124,37 @@ func stats(
 		return nil, nil, fmt.Errorf("failed to fetch jobs: %w", err)
 	}
 
-	// Get all units in the given period
-	// Always add user field as we will need to get total usage
-	urlValues.Add("field", "username")
-	urlValues.Add("field", "num_units")
-
-	// If elapsed is requested we need to get total_time_seconds from usage API resource
-	if slices.Contains(urlValues["field"], "elapsed") {
-		urlValues.Add("field", "total_time_seconds")
-	}
+	// If summary stats are requested, include them
+	var usage []models.Usage
 
-	logger.Debug("Request to fetch usage", "url", usageReqURL, "usage_query", urlValues.Encode())
-
-	usage, err := doRequest[models.Usage](ctx, usageReqURL, urlValues, apiClient)
-	if err != nil {
-		logger.Error("Failed to fetch usage data from CEEMS API server", "err", err)
+	if includeSummaryStats {
+		// Get all units in the given period
+		// Always add user field as we will need to get total usage
+		urlValues.Add("field", "username")
+		urlValues.Add("field", "num_units")
 
-		return nil, nil, fmt.Errorf("failed to fetch usage: %w", err)
-	}
-
-	// If tsData is enabled, get time series data
-	if tsData {
-		// If metrics are not configured, return logging a message
-		if len(config.TSDB.Queries) == 0 {
-			logger.Warn("TSDB queries not configured")
-			fmt.Fprintln(os.Stderr, "time series data not available")
-
-			return units, usage, nil
+		// If elapsed is requested we need to get total_time_seconds from usage API resource
+		if slices.Contains(urlValues["field"], "elapsed") {
+			urlValues.Add("field", "total_time_seconds")
 		}
 
-		logger.Debug("Fetching time series data from TSDB")
+		logger.Debug("Request to fetch usage", "url", usageReqURL.Redacted(), "usage_query", urlValues.Encode())
 
-		err := tsdbData(ctx, logger, config, units, tsDataOut)
+		usage, err = doRequest[models.Usage](ctx, usageReqURL, urlValues, apiClient)
 		if err != nil {
-			logger.Error("failed to fetch time series data", "err", err)
-			fmt.Fprintln(os.Stderr, "failed to fetch time series data")
+			logger.Error("Failed to fetch usage data from CEEMS API server", "err", err)
+
+			return nil, nil, fmt.Errorf("failed to fetch usage: %w", err)
 		}
 	}
 
 	return units, usage, nil
 }
 
 // doRequest does an API request to CEEMS API server and returns response.
-func doRequest[T any](ctx context.Context, reqURL string, urlValues url.Values, client *http.Client) ([]T, error) {
+func doRequest[T any](ctx context.Context, reqURL *url.URL, urlValues url.Values, client *http.Client) ([]T, error) {
 	// Make a new request
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL.String(), nil)
 	if err != nil {
 		return nil, err
 	}