Skip to content

Make CSV importer configurable via ConfigArgParse #166

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MrlixiangWE wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Make CSV importer configurable via ConfigArgParse #166

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion docs/CSV.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,25 @@ tests:
metrics: [metric1, metric2]
csv_options:
delimiter: ','
quotechar: "'"
quote_char: "'"
```

## CSV options

The delimiter and quote character can also be configured globally:

```yaml
csv:
delimiter: ';'
quote_char: "'"
```

The corresponding command-line options are `--csv-delimiter` and
`--csv-quote-char`. They can also be set through the `CSV_DELIMITER` and
`CSV_QUOTE_CHAR` environment variables. Explicit global values override the
`csv_options` values of individual tests; without a global value, existing
per-test settings and defaults are unchanged.

## Example

```bash
Expand Down
2 changes: 1 addition & 1 deletion examples/csv/config/otava.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,4 +24,4 @@ tests:
metrics: [metric1, metric2]
csv_options:
delimiter: ","
quotechar: "'"
quote_char: "'"
16 changes: 12 additions & 4 deletions otava/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from ruamel.yaml import YAML

from otava.bigquery import BigQueryConfig
from otava.csv_options import CsvConfig
from otava.grafana import GrafanaConfig
from otava.graphite import GraphiteConfig
from otava.postgres import PostgresConfig
Expand All @@ -33,6 +34,7 @@

@dataclass
class Config:
csv: CsvConfig
graphite: Optional[GraphiteConfig]
grafana: Optional[GrafanaConfig]
tests: Dict[str, TestConfig]
Expand All @@ -54,7 +56,9 @@ def load_templates(config: Dict) -> Dict[str, Dict]:
return templates


def load_tests(config: Dict, templates: Dict) -> Dict[str, TestConfig]:
def load_tests(
config: Dict, templates: Dict, csv_config: Optional[CsvConfig] = None
) -> Dict[str, TestConfig]:
tests = config.get("tests", {})
if not isinstance(tests, Dict):
raise ConfigError("Property `tests` is not a dictionary")
Expand All @@ -69,7 +73,7 @@ def load_tests(config: Dict, templates: Dict) -> Dict[str, TestConfig]:
except KeyError as e:
raise ConfigError(f"Template {e.args[0]} referenced in test {test_name} not found")
test_config = merge_dict_list(template_list + [test_config])
result[test_name] = create_test_config(test_name, test_config)
result[test_name] = create_test_config(test_name, test_config, csv_config)

return result

Expand All @@ -96,20 +100,22 @@ def load_test_groups(config: Dict, tests: Dict[str, TestConfig]) -> Dict[str, Li


def load_config_from_parser_args(args: configargparse.Namespace) -> Config:
csv_config = CsvConfig.from_parser_args(args)
config_file = getattr(args, "config_file", None)
if config_file is not None:
yaml = YAML(typ="safe")
config = yaml.load(Path(config_file).read_text())

templates = load_templates(config)
tests = load_tests(config, templates)
tests = load_tests(config, templates, csv_config)
groups = load_test_groups(config, tests)
else:
logging.warning("Otava configuration file not found or not specified")
tests = {}
groups = {}

return Config(
csv=csv_config,
graphite=GraphiteConfig.from_parser_args(args),
grafana=GrafanaConfig.from_parser_args(args),
slack=SlackConfig.from_parser_args(args),
Expand All @@ -128,6 +134,7 @@ class NestedYAMLConfigFileParser(configargparse.ConfigFileParser):
"""

CLI_CONFIG_SECTIONS = [
CsvConfig.NAME,
GraphiteConfig.NAME,
GrafanaConfig.NAME,
SlackConfig.NAME,
Expand Down Expand Up @@ -174,7 +181,8 @@ def get_syntax_description(self):


def add_service_option_groups(parser) -> None:
"""Add Graphite, Grafana, Slack, Postgres, and BigQuery option groups to a parser."""
"""Add importer and integration option groups to a parser."""
CsvConfig.add_parser_args(parser.add_argument_group('CSV Options', 'Options for CSV configuration'))
GraphiteConfig.add_parser_args(parser.add_argument_group('Graphite Options', 'Options for Graphite configuration'))
GrafanaConfig.add_parser_args(parser.add_argument_group('Grafana Options', 'Options for Grafana configuration'))
SlackConfig.add_parser_args(parser.add_argument_group('Slack Options', 'Options for Slack configuration'))
Expand Down
33 changes: 33 additions & 0 deletions otava/csv_options.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,39 @@

import enum
from dataclasses import dataclass
from typing import Optional

import configargparse


@dataclass
class CsvConfig:
NAME = "csv"

delimiter: Optional[str] = None
quote_char: Optional[str] = None

@staticmethod
def add_parser_args(arg_group):
arg_group.add_argument(
"--csv-delimiter",
help="CSV delimiter",
env_var="CSV_DELIMITER",
default=configargparse.SUPPRESS,
)
arg_group.add_argument(
"--csv-quote-char",
help="CSV quote character",
env_var="CSV_QUOTE_CHAR",
default=configargparse.SUPPRESS,
)

@staticmethod
def from_parser_args(args):
return CsvConfig(
delimiter=getattr(args, "csv_delimiter", None),
quote_char=getattr(args, "csv_quote_char", None),
)


@dataclass
Expand Down
17 changes: 13 additions & 4 deletions otava/test_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from dataclasses import dataclass
from typing import Dict, List, Optional

from otava.csv_options import CsvOptions
from otava.csv_options import CsvConfig, CsvOptions


@dataclass
Expand Down Expand Up @@ -199,7 +199,9 @@ def fully_qualified_metric_names(self) -> List[str]:
return list(self.metrics.keys())


def create_test_config(name: str, config: Dict) -> TestConfig:
def create_test_config(
name: str, config: Dict, csv_config: Optional[CsvConfig] = None
) -> TestConfig:
"""
Loads properties of a test from a dictionary read from otava's config file
This dictionary must have the `type` property to determine the type of the test.
Expand All @@ -208,7 +210,7 @@ def create_test_config(name: str, config: Dict) -> TestConfig:
"""
test_type = config.get("type")
if test_type == "csv":
return create_csv_test_config(name, config)
return create_csv_test_config(name, config, csv_config)
elif test_type == "graphite":
return create_graphite_test_config(name, config)
elif test_type == "histostat":
Expand All @@ -225,7 +227,9 @@ def create_test_config(name: str, config: Dict) -> TestConfig:
raise TestConfigError(f"Unknown test type {test_type} for test {name}")


def create_csv_test_config(test_name: str, test_info: Dict) -> CsvTestConfig:
def create_csv_test_config(
test_name: str, test_info: Dict, csv_config: Optional[CsvConfig] = None
) -> CsvTestConfig:
csv_options = CsvOptions()
try:
file = test_info["file"]
Expand Down Expand Up @@ -257,6 +261,11 @@ def create_csv_test_config(test_name: str, test_info: Dict) -> CsvTestConfig:
if test_info.get("csv_options"):
csv_options.delimiter = test_info["csv_options"].get("delimiter", ",")
csv_options.quote_char = test_info["csv_options"].get("quote_char", '"')
if csv_config is not None:
if csv_config.delimiter is not None:
csv_options.delimiter = csv_config.delimiter
if csv_config.quote_char is not None:
csv_options.quote_char = csv_config.quote_char
return CsvTestConfig(
test_name,
file,
Expand Down
77 changes: 70 additions & 7 deletions tests/cli_help_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,8 @@ def test_otava_help_output():
assert (
result.stdout
== """\
usage: otava [-h] [--config-file CONFIG_FILE] [--graphite-url GRAPHITE_URL]
usage: otava [-h] [--config-file CONFIG_FILE] [--csv-delimiter CSV_DELIMITER]
[--csv-quote-char CSV_QUOTE_CHAR] [--graphite-url GRAPHITE_URL]
[--grafana-url GRAFANA_URL] [--grafana-user GRAFANA_USER]
[--grafana-password GRAFANA_PASSWORD] [--slack-token SLACK_TOKEN]
[--postgres-hostname POSTGRES_HOSTNAME] [--postgres-port POSTGRES_PORT]
Expand All @@ -73,6 +74,14 @@ def test_otava_help_output():
--config-file CONFIG_FILE
Otava config file path [env var: OTAVA_CONFIG]

CSV Options:
Options for CSV configuration

--csv-delimiter CSV_DELIMITER
CSV delimiter [env var: CSV_DELIMITER]
--csv-quote-char CSV_QUOTE_CHAR
CSV quote character [env var: CSV_QUOTE_CHAR]

Graphite Options:
Options for Graphite configuration

Expand Down Expand Up @@ -147,7 +156,8 @@ def test_otava_analyze_help_output():
magnitude_option = " -M MAGNITUDE, --magnitude MAGNITUDE"

usage_and_options = f"""\
usage: otava analyze [-h] [--config-file CONFIG_FILE] [--graphite-url GRAPHITE_URL]
usage: otava analyze [-h] [--config-file CONFIG_FILE] [--csv-delimiter CSV_DELIMITER]
[--csv-quote-char CSV_QUOTE_CHAR] [--graphite-url GRAPHITE_URL]
[--grafana-url GRAFANA_URL] [--grafana-user GRAFANA_USER]
[--grafana-password GRAFANA_PASSWORD] [--slack-token SLACK_TOKEN]
[--postgres-hostname POSTGRES_HOSTNAME] [--postgres-port POSTGRES_PORT]
Expand Down Expand Up @@ -218,6 +228,14 @@ def test_otava_analyze_help_output():
--orig-edivisive use the original edivisive algorithm with no windowing and weak change
points analysis improvements

CSV Options:
Options for CSV configuration

--csv-delimiter CSV_DELIMITER
CSV delimiter [env var: CSV_DELIMITER]
--csv-quote-char CSV_QUOTE_CHAR
CSV quote character [env var: CSV_QUOTE_CHAR]

Graphite Options:
Options for Graphite configuration

Expand Down Expand Up @@ -278,7 +296,8 @@ def test_otava_list_tests_help_output():
assert (
result.stdout
== """\
usage: otava list-tests [-h] [--config-file CONFIG_FILE] [--graphite-url GRAPHITE_URL]
usage: otava list-tests [-h] [--config-file CONFIG_FILE] [--csv-delimiter CSV_DELIMITER]
[--csv-quote-char CSV_QUOTE_CHAR] [--graphite-url GRAPHITE_URL]
[--grafana-url GRAFANA_URL] [--grafana-user GRAFANA_USER]
[--grafana-password GRAFANA_PASSWORD] [--slack-token SLACK_TOKEN]
[--postgres-hostname POSTGRES_HOSTNAME] [--postgres-port POSTGRES_PORT]
Expand All @@ -298,6 +317,14 @@ def test_otava_list_tests_help_output():
--config-file CONFIG_FILE
Otava config file path [env var: OTAVA_CONFIG]

CSV Options:
Options for CSV configuration

--csv-delimiter CSV_DELIMITER
CSV delimiter [env var: CSV_DELIMITER]
--csv-quote-char CSV_QUOTE_CHAR
CSV quote character [env var: CSV_QUOTE_CHAR]

Graphite Options:
Options for Graphite configuration

Expand Down Expand Up @@ -358,7 +385,8 @@ def test_otava_list_metrics_help_output():
assert (
result.stdout
== """\
usage: otava list-metrics [-h] [--config-file CONFIG_FILE] [--graphite-url GRAPHITE_URL]
usage: otava list-metrics [-h] [--config-file CONFIG_FILE] [--csv-delimiter CSV_DELIMITER]
[--csv-quote-char CSV_QUOTE_CHAR] [--graphite-url GRAPHITE_URL]
[--grafana-url GRAFANA_URL] [--grafana-user GRAFANA_USER]
[--grafana-password GRAFANA_PASSWORD] [--slack-token SLACK_TOKEN]
[--postgres-hostname POSTGRES_HOSTNAME] [--postgres-port POSTGRES_PORT]
Expand All @@ -378,6 +406,14 @@ def test_otava_list_metrics_help_output():
--config-file CONFIG_FILE
Otava config file path [env var: OTAVA_CONFIG]

CSV Options:
Options for CSV configuration

--csv-delimiter CSV_DELIMITER
CSV delimiter [env var: CSV_DELIMITER]
--csv-quote-char CSV_QUOTE_CHAR
CSV quote character [env var: CSV_QUOTE_CHAR]

Graphite Options:
Options for Graphite configuration

Expand Down Expand Up @@ -439,7 +475,8 @@ def test_otava_list_groups_help_output():
assert (
result.stdout
== """\
usage: otava list-groups [-h] [--config-file CONFIG_FILE] [--graphite-url GRAPHITE_URL]
usage: otava list-groups [-h] [--config-file CONFIG_FILE] [--csv-delimiter CSV_DELIMITER]
[--csv-quote-char CSV_QUOTE_CHAR] [--graphite-url GRAPHITE_URL]
[--grafana-url GRAFANA_URL] [--grafana-user GRAFANA_USER]
[--grafana-password GRAFANA_PASSWORD] [--slack-token SLACK_TOKEN]
[--postgres-hostname POSTGRES_HOSTNAME] [--postgres-port POSTGRES_PORT]
Expand All @@ -455,6 +492,14 @@ def test_otava_list_groups_help_output():
--config-file CONFIG_FILE
Otava config file path [env var: OTAVA_CONFIG]

CSV Options:
Options for CSV configuration

--csv-delimiter CSV_DELIMITER
CSV delimiter [env var: CSV_DELIMITER]
--csv-quote-char CSV_QUOTE_CHAR
CSV quote character [env var: CSV_QUOTE_CHAR]

Graphite Options:
Options for Graphite configuration

Expand Down Expand Up @@ -515,7 +560,8 @@ def test_otava_remove_annotations_help_output():
assert (
result.stdout
== """\
usage: otava remove-annotations [-h] [--config-file CONFIG_FILE] [--graphite-url GRAPHITE_URL]
usage: otava remove-annotations [-h] [--config-file CONFIG_FILE] [--csv-delimiter CSV_DELIMITER]
[--csv-quote-char CSV_QUOTE_CHAR] [--graphite-url GRAPHITE_URL]
[--grafana-url GRAFANA_URL] [--grafana-user GRAFANA_USER]
[--grafana-password GRAFANA_PASSWORD] [--slack-token SLACK_TOKEN]
[--postgres-hostname POSTGRES_HOSTNAME]
Expand All @@ -537,6 +583,14 @@ def test_otava_remove_annotations_help_output():
Otava config file path [env var: OTAVA_CONFIG]
--force don't ask questions, just do it

CSV Options:
Options for CSV configuration

--csv-delimiter CSV_DELIMITER
CSV delimiter [env var: CSV_DELIMITER]
--csv-quote-char CSV_QUOTE_CHAR
CSV quote character [env var: CSV_QUOTE_CHAR]

Graphite Options:
Options for Graphite configuration

Expand Down Expand Up @@ -597,7 +651,8 @@ def test_otava_validate_help_output():
assert (
result.stdout
== """\
usage: otava validate [-h] [--config-file CONFIG_FILE] [--graphite-url GRAPHITE_URL]
usage: otava validate [-h] [--config-file CONFIG_FILE] [--csv-delimiter CSV_DELIMITER]
[--csv-quote-char CSV_QUOTE_CHAR] [--graphite-url GRAPHITE_URL]
[--grafana-url GRAFANA_URL] [--grafana-user GRAFANA_USER]
[--grafana-password GRAFANA_PASSWORD] [--slack-token SLACK_TOKEN]
[--postgres-hostname POSTGRES_HOSTNAME] [--postgres-port POSTGRES_PORT]
Expand All @@ -613,6 +668,14 @@ def test_otava_validate_help_output():
--config-file CONFIG_FILE
Otava config file path [env var: OTAVA_CONFIG]

CSV Options:
Options for CSV configuration

--csv-delimiter CSV_DELIMITER
CSV delimiter [env var: CSV_DELIMITER]
--csv-quote-char CSV_QUOTE_CHAR
CSV quote character [env var: CSV_QUOTE_CHAR]

Graphite Options:
Options for Graphite configuration

Expand Down
Loading