Add ooni clickhouse users

ansible/group_vars/clickhouse/vars.yml

@@ -165,6 +165,7 @@ clickhouse_distributed_ddl:
   cleanup_delay_period: 60
   max_tasks_in_queue: 1000
 
+clickhouse_role_manage_settings_profiles: True
 clickhouse_default_profiles:
   default:
     readonly: 2
@@ -226,48 +227,147 @@ clickhouse_default_users:
     profile: write
     quota: default
 
-clickhouse_role_manage_users: true
+clickhouse_role_manage_users: True
 clickhouse_custom_users:
+  - user:
+    name: fastpath
+    password_type: sha256_hash
+    password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') | hash('sha256') }}"
+    networks:
+      - "IP '0.0.0.0/0'"
+    profile:
+      - write
+    quota: "fastpath"
+    databases: [ooni]
+
   - user:
     name: oonimeasurements
-    password_type: sha256_password
-    password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonimeasurements_password', profile='oonidevops_user_prod') }}"
+    password_type: sha256_hash
+    password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonimeasurements_password', profile='oonidevops_user_prod') | hash('sha256') }}"
     networks:
       - "IP '0.0.0.0/0'"
     settings:
-      # 500 MB
-      - "max_memory_usage = 501001000"
+      # 1 GB
+      - "max_memory_usage = 1001001000"
       # 60 seconds
       - "max_execution_time = 30"
+      # 500 GB
+      - "max_bytes_to_read = 501001001000"
+      # 5 B
+      - "max_rows_to_read = 5001001000"
+      # 5s
+      - "timeout_before_checking_execution_speed = 5"
+      # 50k
+      - "max_result_rows = 51000"
     profile:
       - readonly
-    quota: "oonimeasurements"
+    quota: oonimeasurements
+    databases: [ooni, oonitest]
+
+  - user:
+    name: ooniprobe
+    password_type: sha256_hash
+    password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_ooniprobe_password', profile='oonidevops_user_prod') | hash('sha256') }}"
+    networks:
+      - "IP '0.0.0.0/0'"
+    profile:
+      - write
+    quota: ooniprobe
     databases: [ooni]
 
-# TODO: this quota was created by hand since it wasn't working in the idealista playbook
-clickhouse_role_manage_quotas: false
+  - user:
+    name: oonirun
+    password_type: sha256_hash
+    password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonirun_password', profile='oonidevops_user_prod') | hash('sha256') }}"
+    networks:
+      - "IP '0.0.0.0/0'"
+    profile:
+      - write
+    quota: oonirun
+    databases: [ooni]
+
+  - user:
+    name: oonitestlists
+    password_type: sha256_hash
+    password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonitestlists_password', profile='oonidevops_user_prod') | hash('sha256') }}"
+    networks:
+      - "IP '0.0.0.0/0'"
+    profile:
+      - write
+    quota: oonitestlists
+    databases: [ooni]
+
+clickhouse_role_manage_quotas: True
 clickhouse_custom_quotas:
-  # quota over a 10 minute window
   - quota:
     name: oonimeasurements
-    settings:
-      - "INTERVAL 10 minute MAX queries = 12000, MAX errors = 1000, MAX execution_time = 1000"
-    to:
-      - oonimeasurements
+    duration: 600
+    queries: 12000
+    errors: 1000
+    result_rows: 0
+    read_rows: 0
+    execution_time: 1000
 
-clickhouse_role_manage_grants: true
-clickhouse_role_manage_roles: true
+    # no limits set
+  - quota:
+    name: ooniprobe
+    duration: 0
+    queries: 0
+    errors: 0
+    result_rows: 0
+    read_rows: 0
+    execution_time: 0
+
+    # no limits set
+  - quota:
+    name: oonirun
+    duration: 0
+    queries: 0
+    errors: 0
+    result_rows: 0
+    read_rows: 0
+    execution_time: 0
+
+  - quota:
+    name: oonitestlists
+    duration: 600
+    queries: 12000
+    errors: 1000
+    result_rows: 0
+    read_rows: 0
+    execution_time: 1000
+
+clickhouse_role_manage_grants: True
 clickhouse_custom_grants:
 - on:
   databases: [ooni]
   tables: ["*"]
   privileges: [SELECT]
-  to: [oonimeasurements]
+  to: [ooniprobe, oonimeasurements, oonirun, fastpath]
+
+- on:
+  databases: [ooni]
+  tables: [url_priorities]
+  privileges: [INSERT]
+  to: [oonitestlists]
+
+- on:
+  databases: [ooni]
+  tables: [faulty_measurements]
+  privileges: [INSERT]
+  to: [ooniprobe]
+
+- on:
+  databases: [ooni]
+  tables: [fastpath, obs_web, obs_openvpn, jsonl, new_jsonl]
+  privileges: [INSERT]
+  to: [fastpath]
 
 clickhouse_custom_grant_roles:
   - roles: [oonimeasurements]
     to: [oonimeasurements]
 
+clickhouse_role_manage_roles: True
 clickhouse_custom_roles:
   - role:
     name: oonimeasurements

ansible/host_vars/fastpath.dev.ooni.io/vars.yml

@@ -1,5 +1,5 @@
 s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_dev') }}"
-clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest"
+clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest"
 bucket_name: "ooni-data-eu-fra-test"
 # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE
 collector_id: "3"

ansible/host_vars/fastpath.prod.ooni.io/vars.yml

@@ -1,5 +1,5 @@
 s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_prod') }}"
-clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.prod.ooni.io/ooni"
+clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouseproxy.prod.ooni.io/ooni"
 bucket_name: "ooni-data-eu-fra"
 # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE
 collector_id: "1"

ansible/host_vars/fastpath2.prod.ooni.io/vars.yml

@@ -1,5 +1,5 @@
 s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_prod') }}"
-clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@data3.htz-fsn.prod.ooni.nu/ooni"
+clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@data3.htz-fsn.prod.ooni.nu/ooni"
 bucket_name: "ooni-data-eu-fra"
 # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE
 collector_id: "4"

ansible/host_vars/testlist-ec2.dev.ooni.io/vars.yml

@@ -1,3 +1,4 @@
 jwt_encryption_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/ooni_services/jwt_secret', profile='oonidevops_user_dev') }}"
 github_token: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/testlists_github_token', profile='oonidevops_user_dev') }}"
 log_level: "debug"
+clickhouse_url: "clickhouse://oonitestlists:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonitestlists_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/ooni"

ansible/host_vars/testlist-ec2.prod.ooni.io/vars.yml

@@ -1,4 +1,4 @@
 jwt_encryption_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/ooni_services/jwt_secret', profile='oonidevops_user_prod') }}"
 github_token: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/testlists_github_token', profile='oonidevops_user_prod') }}"
 log_level: "info"
-clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.prod.ooni.io/ooni"
+clickhouse_url: "clickhouse://oonitestlists:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonitestlists_password', profile='oonidevops_user_prod') }}@clickhouseproxy.prod.ooni.io/ooni"

ansible/roles/fastpath/defaults/main.yml

@@ -5,4 +5,4 @@ fastpath_user: fastpath
 fastpath_home: "/opt/{{ fastpath_user }}"
 
 # Fastpath settings
-clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest"
+clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest"

tf/environments/dev/main.tf

@@ -237,16 +237,20 @@ resource "aws_secretsmanager_secret_version" "oonipg_url" {
   )
 }
 
-data "aws_ssm_parameter" "clickhouse_readonly_url" {
-  name = "/oonidevops/secrets/clickhouse_readonly_url"
+data "aws_ssm_parameter" "clickhouse_oonimeasurements_url" {
+  name = "/oonidevops/secrets/clickhouse_oonimeasurements_url"
 }
 
-data "aws_ssm_parameter" "clickhouse_readonly_test_url" {
-  name = "/oonidevops/secrets/clickhouse_readonly_test_url"
+data "aws_ssm_parameter" "clickhouse_oonimeasurements_test_url" {
+  name = "/oonidevops/secrets/clickhouse_oonimeasurements_test_url"
 }
 
-data "aws_ssm_parameter" "clickhouse_write_url" {
-  name = "/oonidevops/secrets/clickhouse_write_url"
+data "aws_ssm_parameter" "clickhouse_ooniprobe_url" {
+  name = "/oonidevops/secrets/clickhouse_ooniprobe_url"
+}
+
+data "aws_ssm_parameter" "clickhouse_oonirun_url" {
+  name = "/oonidevops/secrets/clickhouse_oonirun_url"
 }
 
 data "aws_ssm_parameter" "account_id_hashing_key" {
@@ -592,7 +596,7 @@ module "ooniapi_ooniprobe" {
     POSTGRESQL_URL              = data.aws_ssm_parameter.oonipg_url.arn
     JWT_ENCRYPTION_KEY          = data.aws_ssm_parameter.jwt_secret_legacy.arn
     PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn
-    CLICKHOUSE_URL              = data.aws_ssm_parameter.clickhouse_write_url.arn
+    CLICKHOUSE_URL              = data.aws_ssm_parameter.clickhouse_ooniprobe_url.arn
     ANONC_SECRET_KEY            = data.aws_ssm_parameter.anonc_secret_key.arn
   }
 
@@ -973,7 +977,7 @@ module "ooniapi_oonirun" {
     POSTGRESQL_URL              = data.aws_ssm_parameter.oonipg_url.arn
     JWT_ENCRYPTION_KEY          = data.aws_ssm_parameter.jwt_secret.arn
     PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn
-    CLICKHOUSE_URL              = data.aws_ssm_parameter.clickhouse_readonly_url.arn
+    CLICKHOUSE_URL              = data.aws_ssm_parameter.clickhouse_oonirun_url.arn
   }
 
   ooniapi_service_security_groups = [
@@ -1024,7 +1028,6 @@ module "ooniapi_oonifindings" {
     POSTGRESQL_URL              = data.aws_ssm_parameter.oonipg_url.arn
     JWT_ENCRYPTION_KEY          = data.aws_ssm_parameter.jwt_secret.arn
     PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn
-    CLICKHOUSE_URL              = data.aws_ssm_parameter.clickhouse_readonly_url.arn
   }
 
   ooniapi_service_security_groups = [
@@ -1145,7 +1148,7 @@ module "ooniapi_oonimeasurements" {
     POSTGRESQL_URL              = data.aws_ssm_parameter.oonipg_url.arn
     JWT_ENCRYPTION_KEY          = data.aws_ssm_parameter.jwt_secret.arn
     PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn
-    CLICKHOUSE_URL              = data.aws_ssm_parameter.clickhouse_readonly_test_url.arn
+    CLICKHOUSE_URL              = data.aws_ssm_parameter.clickhouse_oonimeasurements_test_url.arn
     ACCOUNT_ID_HASHING_KEY      = data.aws_ssm_parameter.account_id_hashing_key.arn
   }
 

tf/environments/prod/main.tf

@@ -239,16 +239,20 @@ data "aws_ssm_parameter" "oonipg_url" {
   name = "/oonidevops/secrets/ooni-tier0-postgres/postgresql_write_url"
 }
 
-data "aws_ssm_parameter" "clickhouse_readonly_url" {
-  name = "/oonidevops/secrets/clickhouse_readonly_url"
-}
-
 data "aws_ssm_parameter" "clickhouse_oonimeasurements_url" {
   name = "/oonidevops/secrets/clickhouse_oonimeasurements_url"
 }
 
-data "aws_ssm_parameter" "clickhouse_write_url" {
-  name = "/oonidevops/secrets/clickhouse_write_url"
+data "aws_ssm_parameter" "clickhouse_oonimeasurements_test_url" {
+  name = "/oonidevops/secrets/clickhouse_oonimeasurements_test_url"
+}
+
+data "aws_ssm_parameter" "clickhouse_ooniprobe_url" {
+  name = "/oonidevops/secrets/clickhouse_ooniprobe_url"
+}
+
+data "aws_ssm_parameter" "clickhouse_oonirun_url" {
+  name = "/oonidevops/secrets/clickhouse_oonirun_url"
 }
 
 data "aws_ssm_parameter" "account_id_hashing_key" {
@@ -890,7 +894,7 @@ module "ooniapi_ooniprobe" {
     POSTGRESQL_URL              = data.aws_ssm_parameter.oonipg_url.arn
     JWT_ENCRYPTION_KEY          = data.aws_ssm_parameter.jwt_secret.arn
     PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn
-    CLICKHOUSE_URL              = data.aws_ssm_parameter.clickhouse_write_url.arn
+    CLICKHOUSE_URL              = data.aws_ssm_parameter.clickhouse_ooniprobe_url.arn
     ANONC_SECRET_KEY            = data.aws_ssm_parameter.anonc_secret_key.arn
   }
 
@@ -1088,7 +1092,7 @@ module "ooniapi_oonirun" {
     POSTGRESQL_URL              = data.aws_ssm_parameter.oonipg_url.arn
     JWT_ENCRYPTION_KEY          = data.aws_ssm_parameter.jwt_secret.arn
     PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn
-    CLICKHOUSE_URL              = data.aws_ssm_parameter.clickhouse_readonly_url.arn
+    CLICKHOUSE_URL              = data.aws_ssm_parameter.clickhouse_oonirun_url.arn
   }
 
   ooniapi_service_security_groups = [