diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index 7b48ebe1..236ce652 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -165,6 +165,7 @@ clickhouse_distributed_ddl: cleanup_delay_period: 60 max_tasks_in_queue: 1000 +clickhouse_role_manage_settings_profiles: True clickhouse_default_profiles: default: readonly: 2 @@ -226,48 +227,147 @@ clickhouse_default_users: profile: write quota: default -clickhouse_role_manage_users: true +clickhouse_role_manage_users: True clickhouse_custom_users: + - user: + name: fastpath + password_type: sha256_hash + password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') | hash('sha256') }}" + networks: + - "IP '0.0.0.0/0'" + profile: + - write + quota: "fastpath" + databases: [ooni] + - user: name: oonimeasurements - password_type: sha256_password - password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonimeasurements_password', profile='oonidevops_user_prod') }}" + password_type: sha256_hash + password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonimeasurements_password', profile='oonidevops_user_prod') | hash('sha256') }}" networks: - "IP '0.0.0.0/0'" settings: - # 500 MB - - "max_memory_usage = 501001000" + # 1 GB + - "max_memory_usage = 1001001000" # 60 seconds - "max_execution_time = 30" + # 500 GB + - "max_bytes_to_read = 501001001000" + # 5 B + - "max_rows_to_read = 5001001000" + # 5s + - "timeout_before_checking_execution_speed = 5" + # 50k + - "max_result_rows = 51000" profile: - readonly - quota: "oonimeasurements" + quota: oonimeasurements + databases: [ooni, oonitest] + + - user: + name: ooniprobe + password_type: sha256_hash + password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_ooniprobe_password', profile='oonidevops_user_prod') | hash('sha256') }}" + networks: + - "IP '0.0.0.0/0'" + profile: + - write + quota: ooniprobe databases: [ooni] -# TODO: this quota was created by hand since it wasn't working in the idealista playbook -clickhouse_role_manage_quotas: false + - user: + name: oonirun + password_type: sha256_hash + password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonirun_password', profile='oonidevops_user_prod') | hash('sha256') }}" + networks: + - "IP '0.0.0.0/0'" + profile: + - write + quota: oonirun + databases: [ooni] + + - user: + name: oonitestlists + password_type: sha256_hash + password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonitestlists_password', profile='oonidevops_user_prod') | hash('sha256') }}" + networks: + - "IP '0.0.0.0/0'" + profile: + - write + quota: oonitestlists + databases: [ooni] + +clickhouse_role_manage_quotas: True clickhouse_custom_quotas: - # quota over a 10 minute window - quota: name: oonimeasurements - settings: - - "INTERVAL 10 minute MAX queries = 12000, MAX errors = 1000, MAX execution_time = 1000" - to: - - oonimeasurements + duration: 600 + queries: 12000 + errors: 1000 + result_rows: 0 + read_rows: 0 + execution_time: 1000 -clickhouse_role_manage_grants: true -clickhouse_role_manage_roles: true + # no limits set + - quota: + name: ooniprobe + duration: 0 + queries: 0 + errors: 0 + result_rows: 0 + read_rows: 0 + execution_time: 0 + + # no limits set + - quota: + name: oonirun + duration: 0 + queries: 0 + errors: 0 + result_rows: 0 + read_rows: 0 + execution_time: 0 + + - quota: + name: oonitestlists + duration: 600 + queries: 12000 + errors: 1000 + result_rows: 0 + read_rows: 0 + execution_time: 1000 + +clickhouse_role_manage_grants: True clickhouse_custom_grants: - on: databases: [ooni] tables: ["*"] privileges: [SELECT] - to: [oonimeasurements] + to: [ooniprobe, oonimeasurements, oonirun, fastpath] + +- on: + databases: [ooni] + tables: [url_priorities] + privileges: [INSERT] + to: [oonitestlists] + +- on: + databases: [ooni] + tables: [faulty_measurements] + privileges: [INSERT] + to: [ooniprobe] + +- on: + databases: [ooni] + tables: [fastpath, obs_web, obs_openvpn, jsonl, new_jsonl] + privileges: [INSERT] + to: [fastpath] clickhouse_custom_grant_roles: - roles: [oonimeasurements] to: [oonimeasurements] +clickhouse_role_manage_roles: True clickhouse_custom_roles: - role: name: oonimeasurements diff --git a/ansible/host_vars/fastpath.dev.ooni.io/vars.yml b/ansible/host_vars/fastpath.dev.ooni.io/vars.yml index fbd3d273..e60bcb31 100644 --- a/ansible/host_vars/fastpath.dev.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath.dev.ooni.io/vars.yml @@ -1,5 +1,5 @@ s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_dev') }}" -clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest" +clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest" bucket_name: "ooni-data-eu-fra-test" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "3" diff --git a/ansible/host_vars/fastpath.prod.ooni.io/vars.yml b/ansible/host_vars/fastpath.prod.ooni.io/vars.yml index e7210f1f..ffcb68ef 100644 --- a/ansible/host_vars/fastpath.prod.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath.prod.ooni.io/vars.yml @@ -1,5 +1,5 @@ s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_prod') }}" -clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.prod.ooni.io/ooni" +clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouseproxy.prod.ooni.io/ooni" bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "1" diff --git a/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml b/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml index 28e22d69..fa79213e 100644 --- a/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml @@ -1,5 +1,5 @@ s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_prod') }}" -clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@data3.htz-fsn.prod.ooni.nu/ooni" +clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@data3.htz-fsn.prod.ooni.nu/ooni" bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "4" diff --git a/ansible/host_vars/testlist-ec2.dev.ooni.io/vars.yml b/ansible/host_vars/testlist-ec2.dev.ooni.io/vars.yml index 7ab99273..b6497100 100644 --- a/ansible/host_vars/testlist-ec2.dev.ooni.io/vars.yml +++ b/ansible/host_vars/testlist-ec2.dev.ooni.io/vars.yml @@ -1,3 +1,4 @@ jwt_encryption_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/ooni_services/jwt_secret', profile='oonidevops_user_dev') }}" github_token: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/testlists_github_token', profile='oonidevops_user_dev') }}" log_level: "debug" +clickhouse_url: "clickhouse://oonitestlists:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonitestlists_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/ooni" diff --git a/ansible/host_vars/testlist-ec2.prod.ooni.io/vars.yml b/ansible/host_vars/testlist-ec2.prod.ooni.io/vars.yml index 59e4bb4c..fb061292 100644 --- a/ansible/host_vars/testlist-ec2.prod.ooni.io/vars.yml +++ b/ansible/host_vars/testlist-ec2.prod.ooni.io/vars.yml @@ -1,4 +1,4 @@ jwt_encryption_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/ooni_services/jwt_secret', profile='oonidevops_user_prod') }}" github_token: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/testlists_github_token', profile='oonidevops_user_prod') }}" log_level: "info" -clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.prod.ooni.io/ooni" +clickhouse_url: "clickhouse://oonitestlists:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonitestlists_password', profile='oonidevops_user_prod') }}@clickhouseproxy.prod.ooni.io/ooni" diff --git a/ansible/roles/fastpath/defaults/main.yml b/ansible/roles/fastpath/defaults/main.yml index 1a15aea5..2a40e928 100644 --- a/ansible/roles/fastpath/defaults/main.yml +++ b/ansible/roles/fastpath/defaults/main.yml @@ -5,4 +5,4 @@ fastpath_user: fastpath fastpath_home: "/opt/{{ fastpath_user }}" # Fastpath settings -clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest" \ No newline at end of file +clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest" diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index c569935b..09590756 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -237,16 +237,20 @@ resource "aws_secretsmanager_secret_version" "oonipg_url" { ) } -data "aws_ssm_parameter" "clickhouse_readonly_url" { - name = "/oonidevops/secrets/clickhouse_readonly_url" +data "aws_ssm_parameter" "clickhouse_oonimeasurements_url" { + name = "/oonidevops/secrets/clickhouse_oonimeasurements_url" } -data "aws_ssm_parameter" "clickhouse_readonly_test_url" { - name = "/oonidevops/secrets/clickhouse_readonly_test_url" +data "aws_ssm_parameter" "clickhouse_oonimeasurements_test_url" { + name = "/oonidevops/secrets/clickhouse_oonimeasurements_test_url" } -data "aws_ssm_parameter" "clickhouse_write_url" { - name = "/oonidevops/secrets/clickhouse_write_url" +data "aws_ssm_parameter" "clickhouse_ooniprobe_url" { + name = "/oonidevops/secrets/clickhouse_ooniprobe_url" +} + +data "aws_ssm_parameter" "clickhouse_oonirun_url" { + name = "/oonidevops/secrets/clickhouse_oonirun_url" } data "aws_ssm_parameter" "account_id_hashing_key" { @@ -592,7 +596,7 @@ module "ooniapi_ooniprobe" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret_legacy.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_ooniprobe_url.arn ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.arn } @@ -973,7 +977,7 @@ module "ooniapi_oonirun" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonirun_url.arn } ooniapi_service_security_groups = [ @@ -1024,7 +1028,6 @@ module "ooniapi_oonifindings" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.arn } ooniapi_service_security_groups = [ @@ -1145,7 +1148,7 @@ module "ooniapi_oonimeasurements" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_test_url.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_test_url.arn ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.arn } diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 4c343293..7c26100e 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -239,16 +239,20 @@ data "aws_ssm_parameter" "oonipg_url" { name = "/oonidevops/secrets/ooni-tier0-postgres/postgresql_write_url" } -data "aws_ssm_parameter" "clickhouse_readonly_url" { - name = "/oonidevops/secrets/clickhouse_readonly_url" -} - data "aws_ssm_parameter" "clickhouse_oonimeasurements_url" { name = "/oonidevops/secrets/clickhouse_oonimeasurements_url" } -data "aws_ssm_parameter" "clickhouse_write_url" { - name = "/oonidevops/secrets/clickhouse_write_url" +data "aws_ssm_parameter" "clickhouse_oonimeasurements_test_url" { + name = "/oonidevops/secrets/clickhouse_oonimeasurements_test_url" +} + +data "aws_ssm_parameter" "clickhouse_ooniprobe_url" { + name = "/oonidevops/secrets/clickhouse_ooniprobe_url" +} + +data "aws_ssm_parameter" "clickhouse_oonirun_url" { + name = "/oonidevops/secrets/clickhouse_oonirun_url" } data "aws_ssm_parameter" "account_id_hashing_key" { @@ -890,7 +894,7 @@ module "ooniapi_ooniprobe" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_ooniprobe_url.arn ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.arn } @@ -1088,7 +1092,7 @@ module "ooniapi_oonirun" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonirun_url.arn } ooniapi_service_security_groups = [