From c97594f60495145996dd3c171fe8a5d2698622f9 Mon Sep 17 00:00:00 2001 From: Wyatt Pearsall Date: Mon, 17 Aug 2026 13:48:32 -0700 Subject: [PATCH 1/6] Remove narratives --- complaint_search/defaults.py | 11 - complaint_search/es_builders.py | 341 -------- complaint_search/es_interface.py | 135 --- complaint_search/serializer.py | 80 +- complaint_search/tests/test_es_interface.py | 5 - .../tests/test_es_interface_states.py | 70 -- .../tests/test_es_interface_trends.py | 179 ---- complaint_search/tests/test_serializer.py | 96 +-- complaint_search/tests/test_views_search.py | 14 - complaint_search/tests/test_views_states.py | 84 -- complaint_search/tests/test_views_trends.py | 69 -- complaint_search/urls.py | 4 - complaint_search/views.py | 49 -- setup.py | 1 - swagger-config.yaml | 787 ------------------ 15 files changed, 2 insertions(+), 1923 deletions(-) delete mode 100644 complaint_search/tests/test_es_interface_states.py delete mode 100644 complaint_search/tests/test_es_interface_trends.py delete mode 100644 complaint_search/tests/test_views_states.py delete mode 100644 complaint_search/tests/test_views_trends.py delete mode 100644 swagger-config.yaml diff --git a/complaint_search/defaults.py b/complaint_search/defaults.py index 3256e212..23b731be 100644 --- a/complaint_search/defaults.py +++ b/complaint_search/defaults.py @@ -9,8 +9,6 @@ AGG_COMPANY_DEFAULT = 6500 AGG_ZIPCODE_DEFAULT = 26000 AGG_STATE_DEFAULT = 100 -AGG_STATE_PRODUCT_DEFAULT = 5 -AGG_STATE_ISSUE_DEFAULT = 5 AGG_ISSUE_DEFAULT = 200 AGG_SUBISSUE_DEFAULT = 250 AGG_PRODUCT_DEFAULT = 30 @@ -19,13 +17,11 @@ # Pagination batch is the number of results we paginate at a time. # Max pagination depth is the farthest we'll paginate – 100 batches. # The default result size matches the front-end default for users. -# The trend_depth default limits display to 5 items in some Trends contexts. PAGINATION_BATCH = 100 MAX_DOWNLOAD_SIZE = 100000 MAX_PAGINATION_DEPTH = 10000 RESULT_SIZE_DEFAULT = 25 RESULT_SIZE_OPTIONS = [10, 50, 100] -TREND_DEPTH_DEFAULT = 5 PARAMS = { "format": "default", @@ -94,10 +90,3 @@ FORMAT_CONTENT_TYPE_MAP = { "csv": "text/csv", } - -DATA_SUB_LENS_MAP = { - "product": ("sub_product", "issue", "company", "tags"), - "issue": ("product", "sub_issue", "company", "tags"), - "company": ("product", "issue", "tags"), - "tags": ("product", "issue", "company"), -} diff --git a/complaint_search/es_builders.py b/complaint_search/es_builders.py index 1920750a..63a0eea8 100644 --- a/complaint_search/es_builders.py +++ b/complaint_search/es_builders.py @@ -8,18 +8,14 @@ AGG_ISSUE_DEFAULT, AGG_PRODUCT_DEFAULT, AGG_STATE_DEFAULT, - AGG_STATE_ISSUE_DEFAULT, - AGG_STATE_PRODUCT_DEFAULT, AGG_SUBISSUE_DEFAULT, AGG_SUBPRODUCT_DEFAULT, AGG_ZIPCODE_DEFAULT, - DATA_SUB_LENS_MAP, DELIMITER, EXCLUDE_PREFIX, EXPORT_FORMATS, PARAMS, SOURCE_FIELDS, - TREND_DEPTH_DEFAULT, ) @@ -63,7 +59,6 @@ class BaseBuilder(object): "company", "company_public_response", "company_response", - "has_narrative", "issue", "product", "state", @@ -433,339 +428,3 @@ def build(self): } ) return aggs - - -class StateAggregationBuilder(BaseBuilder): - _AGG_FIELDS = ( - "issue", - "product", - "state", - ) - - _AGG_SIZES = { - "state": AGG_STATE_DEFAULT, # 100 - "product.raw": AGG_STATE_PRODUCT_DEFAULT, # 5 - "issue.raw": AGG_STATE_ISSUE_DEFAULT, # 5 - } - - _ES_CHILD_AGG_MAP = { - "product.raw": "sub-product", - "sub_product.raw": "product", - "issue.raw": "sub-issue", - "sub_issue.raw": "issue", - "tags": "tags", - } - - def __init__(self): - BaseBuilder.__init__(self) - self.include_clauses = None - self.exclude_clauses = None - self.exclude = [] - - def add_exclude(self, field_name_list): - self.exclude += field_name_list - - def build_one(self, field_name): - # Lazy initialization - if not self.include_clauses: - ( - self.include_clauses, - self.exclude_clauses, - ) = self._build_clauses_dictionary() - - field_aggs = {"filter": {}} - - es_field_name = self._OPTIONAL_FILTERS_PARAM_TO_ES_MAP.get( - field_name, field_name - ) - es_child_name = self._OPTIONAL_FILTERS_PARAM_TO_ES_MAP.get( - self._OPTIONAL_FILTERS_CHILD_MAP.get(field_name) - ) - - field_aggs["aggs"] = { - field_name: { - "terms": { - "field": es_field_name, - "size": self._AGG_SIZES.get(es_field_name), - }, - "aggs": { - self._ES_CHILD_AGG_MAP.get(es_child_name): { - "terms": {"field": es_child_name} - } - }, - } - } - - # Get top product & issue for each state. - # Don't apply state filter to itself - if field_name == "state": - field_aggs["aggs"]["state"]["aggs"] = { - "product": {"terms": {"field": "product.raw"}}, - "issue": {"terms": {"field": "issue.raw"}}, - } - - filtered_includes = copy.deepcopy(self.include_clauses) - if "state" in filtered_includes: - del filtered_includes["state"] - - field_aggs["filter"] = self._build_dsl_filter( - filtered_includes, self.exclude_clauses - ) - else: - field_aggs["filter"] = self._build_dsl_filter( - self.include_clauses, self.exclude_clauses - ) - - return field_aggs - - def build(self): - aggs = {} - - agg_fields = self._AGG_FIELDS - if self.exclude: - agg_fields = [ - field_name - for field_name in self._AGG_FIELDS - if field_name not in self.exclude - ] - for field_name in agg_fields: - aggs[field_name] = self.build_one(field_name) - - return aggs - - -class LensAggregationBuilder(BaseBuilder): - _ES_CHILD_AGG_MAP = { - "product.raw": "sub-product", - "sub_product.raw": "product", - "issue.raw": "sub-issue", - "sub_issue.raw": "issue", - "tags": "tags", - } - - def __init__(self): - super(LensAggregationBuilder, self).__init__() - self._include_clauses = None - self.exclude_clauses = None - self.exclude = [] - - @property - def include_clauses(self): - if not self._include_clauses: - ( - self._include_clauses, - self.exclude_clauses, - ) = self._build_clauses_dictionary() - return self._include_clauses - - def add_exclude(self, field_name_list): - self.exclude += field_name_list - - def build_histogram(self, agg_name, interval, include_date_filter): - agg = { - "aggs": { - agg_name: { - "date_histogram": { - "field": "date_received", - "interval": interval, - } - } - } - } - - # Add filter clauses - agg["filter"] = self._build_dsl_filter( - self.include_clauses, - self.exclude_clauses, - include_dates=include_date_filter, - single_not_clause=False, - ) - - return agg - - def date_extreme(self, extreme): - return { - extreme: { - "field": "date_received", - "format": "yyyy-MM-dd'T'12:00:00-05:00", - } - } - - -class TrendsAggregationBuilder(LensAggregationBuilder): - _AGG_FIELDS = ("issue", "tags", "product") - - _AGG_HEADING_MAP = { - "issue": "issue", - "product": "product", - "sub_product": "sub-product", - "sub_issue": "sub-issue", - "tags": "tags", - } - - def percent_change_agg(self, es_field_name, interval, trend_depth): - return { - "terms": {"size": trend_depth, "field": es_field_name}, - "aggs": { - "trend_period": { - "date_histogram": { - "field": "date_received", - "interval": interval, - }, - "aggs": { - "interval_diff": { - "serial_diff": {"buckets_path": "_count"} - } - }, - } - }, - } - - def get_agg_heading(self, field_name): - return self._AGG_HEADING_MAP.get(field_name, field_name) - - def agg_setup(self, field_name, agg_heading_name, interval): - field_aggs = {"aggs": {}} - - es_field_name = self._OPTIONAL_FILTERS_PARAM_TO_ES_MAP.get( - field_name, field_name - ) - - field_aggs["aggs"][agg_heading_name] = self.percent_change_agg( - es_field_name, interval, self.params["trend_depth"] - ) - - # Add the filters - field_aggs["filter"] = self._build_dsl_filter( - self.include_clauses, self.exclude_clauses, single_not_clause=False - ) - - return field_aggs - - def build_one_overview(self, field_name, agg_heading_name, interval): - field_aggs = self.agg_setup(field_name, agg_heading_name, interval) - - if field_name in self._OPTIONAL_FILTERS_CHILD_MAP: - es_child_name = self._OPTIONAL_FILTERS_PARAM_TO_ES_MAP.get( - self._OPTIONAL_FILTERS_CHILD_MAP.get(field_name) - ) - child_agg_name = self._ES_CHILD_AGG_MAP.get(es_child_name) - field_aggs["aggs"][agg_heading_name]["aggs"][ - child_agg_name - ] = self.percent_change_agg( - es_child_name, interval, self.params["trend_depth"] - ) - - return field_aggs - - def build_one_lens(self, field_name, agg_heading_name, interval, sub_lens): - field_aggs = self.agg_setup(field_name, agg_heading_name, interval) - - es_child_name = self._OPTIONAL_FILTERS_PARAM_TO_ES_MAP.get(sub_lens) - child_agg_name = self._ES_CHILD_AGG_MAP.get(es_child_name) - field_aggs["aggs"][agg_heading_name]["aggs"][ - child_agg_name - ] = self.percent_change_agg( - es_child_name, interval, self.params["sub_lens_depth"] - ) - - return field_aggs - - def focus_filter(self): - return { - "term": { - self._OPTIONAL_FILTERS_PARAM_TO_ES_MAP.get( - self.params["lens"] - ): self.params["focus"] - } - } - - def build_one_focus(self, field_name, agg_heading_name, interval): - - field_aggs = self.agg_setup(field_name, agg_heading_name, interval) - - field_aggs["filter"]["bool"]["must"].append(self.focus_filter()) - - return field_aggs - - def build(self): - if not self.params.get("trend_depth"): - self.params["trend_depth"] = TREND_DEPTH_DEFAULT - - aggs = {} - - aggs["dateRangeArea"] = self.build_histogram( - "dateRangeArea", self.params["trend_interval"], True - ) - aggs["dateRangeBrush"] = self.build_histogram( - "dateRangeBrush", self.params["trend_interval"], False - ) - - if self.params["lens"] == "overview": - # Reset default for overview row charts - - for field_name in self._AGG_FIELDS: - if field_name not in self.exclude: - agg_heading_name = self.get_agg_heading(field_name) - - aggs[agg_heading_name] = self.build_one_overview( - field_name, - agg_heading_name, - self.params["trend_interval"], - ) - elif "focus" in self.params: - self.params["trend_depth"] = 10 - for field_name in DATA_SUB_LENS_MAP.get(self.params["lens"]) + ( - self.params["lens"], - ): - agg_heading_name = self.get_agg_heading(field_name) - - aggs[agg_heading_name] = self.build_one_focus( - field_name, agg_heading_name, self.params["trend_interval"] - ) - - aggs["dateRangeArea"]["filter"]["bool"]["must"].append( - self.focus_filter() - ) - aggs["dateRangeBrush"]["filter"]["bool"]["must"].append( - self.focus_filter() - ) - - else: - agg_heading_name = self.get_agg_heading(self.params["lens"]) - - aggs[agg_heading_name] = self.build_one_lens( - self.params["lens"], - agg_heading_name, - self.params["trend_interval"], - self.params["sub_lens"], - ) - - aggs["min_date"] = self.date_extreme("min") - aggs["max_date"] = self.date_extreme("max") - return aggs - - -class DateRangeBucketsBuilder(BaseBuilder): - def build(self): - agg = { - "dateRangeBuckets": { - "aggs": { - "dateRangeBuckets": { - "date_histogram": { - "field": "date_received", - "calendar_interval": self.params.get( - "trend_interval", TREND_DEPTH_DEFAULT - ), - } - } - } - } - } - - # Add date filter clause - agg["dateRangeBuckets"]["filter"] = self._build_dsl_filter( - {}, {}, include_dates=True, single_not_clause=False - ) - - return agg diff --git a/complaint_search/es_interface.py b/complaint_search/es_interface.py index 90837527..016e947f 100644 --- a/complaint_search/es_interface.py +++ b/complaint_search/es_interface.py @@ -17,11 +17,8 @@ ) from complaint_search.es_builders import ( AggregationBuilder, - DateRangeBucketsBuilder, PostFilterBuilder, SearchBuilder, - StateAggregationBuilder, - TrendsAggregationBuilder, ) from complaint_search.export import OpenSearchExporter @@ -39,27 +36,6 @@ _COMPLAINT_ES_INDEX = os.environ.get("COMPLAINT_ES_INDEX", "complaint-index") -# ----------------------------------------------------------------------------- -# Trends Operations -# ----------------------------------------------------------------------------- - - -def extract_date(agg_term, default_value): - return agg_term.get("value_as_string", default_value) - - -def build_trend_meta(response): - meta = {} - - if "max_date" in response["aggregations"]: - date_max = extract_date(response["aggregations"]["max_date"], None) - date_min = extract_date(response["aggregations"]["min_date"], None) - meta["date_min"] = date_min - meta["date_max"] = date_max - - return meta - - # PAGINATION_BATCH = 100 # MAX_PAGINATION_DEPTH = 10000 def get_pagination_query_size(page, user_batch_size): @@ -91,51 +67,6 @@ def get_break_points(hits, size): return break_points -# Find sub_agg name if it exists in order to filter percent change -def get_sug_agg_key_if_exists(agg): - agg_keys_exclude = ("trend_period", "key", "doc_count") - for key in agg.keys(): - if key not in agg_keys_exclude: - return key - return None - - -# Filter out all but the most recent buckets in sub agg -# for the Percent Change on chart -def process_trend_aggregations(aggregations): - trend_charts = ("product", "sub-product", "issue", "sub-issue", "tags") - - for agg_name in trend_charts: - if agg_name in aggregations: - agg_buckets = aggregations[agg_name][agg_name]["buckets"] - for sub_agg in agg_buckets: - sub_agg["trend_period"]["buckets"] = sorted( - sub_agg["trend_period"]["buckets"], - key=lambda k: k["key_as_string"], - reverse=True, - ) - sub_agg_name = get_sug_agg_key_if_exists(sub_agg) - if sub_agg_name: - for sub_sub_agg in sub_agg[sub_agg_name]["buckets"]: - sub_sub_agg["trend_period"]["buckets"] = sorted( - sub_sub_agg["trend_period"]["buckets"], - key=lambda k: k["key_as_string"], - reverse=True, - )[1:2] - return aggregations - - -# Process the response from a trends query -def process_trends_response(response): - response["aggregations"] = process_trend_aggregations( - response["aggregations"] - ) - - response["_meta"] = build_trend_meta(response) - - return response - - def _get_es(): global _ES_INSTANCE if _ES_INSTANCE is None: @@ -428,69 +359,3 @@ def document(complaint_id): doc_query = {"query": {"term": {"_id": complaint_id}}} res = _get_es().search(index=_COMPLAINT_ES_INDEX, body=doc_query) return res - - -def states_agg(agg_exclude=None, **kwargs): - params = copy.deepcopy(PARAMS) - params.update(**kwargs) - params.update({"size": 0}) - search_builder = SearchBuilder() - search_builder.add(**params) - body = search_builder.build() - aggregation_builder = StateAggregationBuilder() - aggregation_builder.add(**params) - if agg_exclude: - aggregation_builder.add_exclude(agg_exclude) - body["aggs"] = aggregation_builder.build() - body["track_total_hits"] = True - log.info( - "Calling %s/%s/_search with %s", - _ES_URL, - _COMPLAINT_ES_INDEX, - body, - ) - log.info("API params were %s", params) - res = _get_es().search(index=_COMPLAINT_ES_INDEX, body=body) - return res - - -def trends(agg_exclude=None, **kwargs): - params = copy.deepcopy(PARAMS) - params.update(**kwargs) - params.update(size=0) - search_builder = SearchBuilder() - search_builder.add(**params) - body = search_builder.build() - - res_trends = None - - aggregation_builder = TrendsAggregationBuilder() - aggregation_builder.add(**params) - if agg_exclude: - aggregation_builder.add_exclude(agg_exclude) - body["aggs"] = aggregation_builder.build() - body["track_total_hits"] = True - - res_trends = _get_es().search(index=_COMPLAINT_ES_INDEX, body=body) - - res_date_buckets = None - - date_bucket_body = copy.deepcopy(body) - date_bucket_body["query"] = {"match_all": {}} - - date_range_buckets_builder = DateRangeBucketsBuilder() - date_range_buckets_builder.add(**params) - date_bucket_body["aggs"] = date_range_buckets_builder.build() - - res_date_buckets = _get_es().search( - index=_COMPLAINT_ES_INDEX, body=date_bucket_body - ) - - res_trends = process_trends_response(res_trends) - res_trends["aggregations"]["dateRangeBuckets"] = res_date_buckets[ - "aggregations" - ]["dateRangeBuckets"] - - res_trends["aggregations"]["dateRangeBuckets"]["body"] = date_bucket_body - - return res_trends diff --git a/complaint_search/serializer.py b/complaint_search/serializer.py index d88f2126..e9330d12 100644 --- a/complaint_search/serializer.py +++ b/complaint_search/serializer.py @@ -1,7 +1,7 @@ from localflavor.us.us_states import STATE_CHOICES from rest_framework import serializers -from complaint_search.defaults import DATA_SUB_LENS_MAP, PARAMS +from complaint_search.defaults import PARAMS class SearchInputSerializer(serializers.Serializer): @@ -184,81 +184,3 @@ class SuggestInputSerializer(serializers.Serializer): class SuggestFilterInputSerializer(SearchInputSerializer): text = serializers.CharField(max_length=100, required=True) - - -class TrendsInputSerializer(SearchInputSerializer): - # ----------------------------------------------------------------------------- - # Constants - # - - YEARLY = "year" - QUARTERLY = "quarter" - MONTHLY = "month" - WEEKLY = "week" - DAILY = "day" - - INTERVAL_CHOICES = ( - (YEARLY, "Yearly Interval"), - (QUARTERLY, "Quarterly Interval"), - (MONTHLY, "Monthly Interval"), - (WEEKLY, "Weekly Interval"), - (DAILY, "Daily Interval"), - ) - - # Data Lens Choices - OVERVIEW = "overview" - PRODUCT = "product" - SUBPRODUCT = "sub_product" - ISSUE = "issue" - SUBISSUE = "sub_issue" - COMPANY = "company" - TAGS = "tags" - - DATA_LENS_CHOICES = ( - (OVERVIEW, "Overview Lens"), - (PRODUCT, "Product Lens"), - (ISSUE, "Issue Lens"), - (COMPANY, "Company Lens"), - (TAGS, "Tags Lens"), - ) - - focus = serializers.CharField( - min_length=1, max_length=10000, required=False - ) - trend_interval = serializers.ChoiceField(INTERVAL_CHOICES) - trend_depth = serializers.IntegerField( - min_value=5, max_value=10000000, default=5 - ) - sub_lens_depth = serializers.IntegerField( - min_value=5, max_value=10000000, default=5 - ) - lens = serializers.ChoiceField(DATA_LENS_CHOICES) - sub_lens = serializers.CharField( - min_length=5, max_length=100, required=False - ) - - def validate(self, data): - if ( - "focus" not in data - and "sub_lens" not in data - and not data["lens"] == "overview" - ): - raise serializers.ValidationError( - "Either Focus or Sub-lens is required for lens '{}'." - " Valid sub-lens are: {}".format( - data["lens"], DATA_SUB_LENS_MAP[data["lens"]] - ) - ) - - if "sub_lens" in data and not data["lens"] == "overview": - if not data["sub_lens"] in DATA_SUB_LENS_MAP[data["lens"]]: - raise serializers.ValidationError( - "'{}' is not a valid sub-lens for '{}'." - " Valid sub-lens are: {}".format( - data["sub_lens"], - data["lens"], - DATA_SUB_LENS_MAP[data["lens"]], - ) - ) - - return data diff --git a/complaint_search/tests/test_es_interface.py b/complaint_search/tests/test_es_interface.py index 8784255d..bdfb688a 100644 --- a/complaint_search/tests/test_es_interface.py +++ b/complaint_search/tests/test_es_interface.py @@ -462,11 +462,6 @@ def test_search_with_tags__valid(self): "search_with_tags__valid", tags=["Older American", "Servicemember"] ) - def test_search_with_has_narrative__valid(self): - self.request_test( - "search_with_has_narrative__valid", has_narrative=["true"] - ) - @mock.patch("requests.get", ok=True, content="RGET_OK") def test_search_no_highlight__valid(self, mock_rget): self.request_test("search_no_highlight__valid", no_highlight=True) diff --git a/complaint_search/tests/test_es_interface_states.py b/complaint_search/tests/test_es_interface_states.py deleted file mode 100644 index 7b62d626..00000000 --- a/complaint_search/tests/test_es_interface_states.py +++ /dev/null @@ -1,70 +0,0 @@ -from unittest import mock - -from django.test import TestCase - -from opensearchpy import OpenSearch - -from complaint_search.es_interface import states_agg -from complaint_search.tests.es_interface_test_helpers import ( - assertBodyEqual, - load, -) - - -class EsInterfaceTestStates(TestCase): - @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") - @mock.patch.object(OpenSearch, "search") - @mock.patch.object(OpenSearch, "count") - def test_states__valid(self, mock_count, mock_search): - mock_count.return_value = {"count": 100} - body = load("states_agg__valid") - mock_search.return_value = { - "hits": {"total": {"value": 100, "relation": "eq"}} - } - res = states_agg() - self.assertEqual(len(mock_search.call_args), 2) - self.assertEqual(0, len(mock_search.call_args[0])) - self.assertEqual(2, len(mock_search.call_args[1])) - assertBodyEqual(body, mock_search.call_args_list[0][1]["body"]) - self.assertEqual(mock_search.call_args[1]["index"], "INDEX") - self.assertEqual(res["hits"]["total"]["relation"], "eq") - - @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") - @mock.patch.object(OpenSearch, "search") - @mock.patch.object(OpenSearch, "count") - def test_states_exclude__valid(self, mock_count, mock_search): - body = load("states_agg__valid") - mock_search.return_value = { - "hits": {"total": {"value": 100, "relation": "eq"}} - } - res = states_agg(["zip_code"]) - self.assertEqual(len(mock_search.call_args), 2) - self.assertEqual(0, len(mock_search.call_args[0])) - self.assertEqual(2, len(mock_search.call_args[1])) - assertBodyEqual(body, mock_search.call_args_list[0][1]["body"]) - self.assertEqual(mock_search.call_args[1]["index"], "INDEX") - self.assertEqual(res["hits"]["total"]["relation"], "eq") - - @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") - @mock.patch.object(OpenSearch, "search") - def test_states_date_filters__valid(self, mock_search): - params = { - "date_received_min": "2020-01-01", - "date_received_max": "2020-05-05", - "company_received_min": "2020-01-01", - "company_received_max": "2020-05-05", - "state": ["VA"], - "size": 0, - } - body = load("states_date_filters__valid") - - mock_search.return_value = { - "hits": {"total": {"value": 100, "relation": "eq"}} - } - res = states_agg(**params) - self.assertEqual(len(mock_search.call_args), 2) - self.assertEqual(0, len(mock_search.call_args[0])) - self.assertEqual(2, len(mock_search.call_args[1])) - assertBodyEqual(body, mock_search.call_args_list[0][1]["body"]) - self.assertEqual(mock_search.call_args[1]["index"], "INDEX") - self.assertEqual(res, mock_search.return_value) diff --git a/complaint_search/tests/test_es_interface_trends.py b/complaint_search/tests/test_es_interface_trends.py deleted file mode 100644 index 8266c042..00000000 --- a/complaint_search/tests/test_es_interface_trends.py +++ /dev/null @@ -1,179 +0,0 @@ -from unittest import mock - -from django.test import TestCase - -from opensearchpy import OpenSearch - -from complaint_search.defaults import AGG_EXCLUDE_FIELDS -from complaint_search.es_interface import trends -from complaint_search.tests.es_interface_test_helpers import load - - -class EsInterfaceTestTrends(TestCase): - @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") - @mock.patch.object(OpenSearch, "count") - @mock.patch.object(OpenSearch, "search") - def test_trends_default_params__valid(self, mock_search, mock_count): - trends_params = {"lens": "overview", "trend_interval": "year"} - body = load("trends_default_params__valid") - mock_count.return_value = {"count": body["hits"]["total"]["value"]} - mock_search.return_value = body - res = trends(**trends_params) - self.assertEqual(len(mock_search.call_args), 2) - self.assertEqual(mock_search.call_args[1]["index"], "INDEX") - self.assertEqual(body, res) - - @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") - @mock.patch.object(OpenSearch, "count") - @mock.patch.object(OpenSearch, "search") - def test_trends_sub_lens_product__valid(self, mock_search, mock_count): - trends_params = { - "lens": "product", - "trend_interval": "year", - "sub_lens": "sub_product", - "trend_depth": 5, - "sub_lens_depth": 5, - } - body = load("trends_sub_lens_product__valid") - mock_count.return_value = {"count": body["hits"]["total"]["value"]} - mock_search.return_value = body - - res = trends(**trends_params) - self.assertEqual(len(mock_search.call_args), 2) - self.assertEqual(mock_search.call_args[1]["index"], "INDEX") - self.assertEqual(body, res) - - @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") - @mock.patch.object(OpenSearch, "count") - @mock.patch.object(OpenSearch, "search") - def test_trends_exclude_and_date_filters__valid( - self, mock_search, mock_count - ): - trends_params = { - "lens": "overview", - "trend_interval": "year", - "date_received_min": "2019-01-01", - "date_received_max": "2020-01-01", - "company_received_min": "2019-01-01", - "company_received_max": "2020-01-01", - } - body = load("trends_exclude_and_date_filters__valid") - mock_count.return_value = {"count": body["hits"]["total"]["value"]} - mock_search.return_value = body - - res = trends(agg_exclude=["zip_code"], **trends_params) - self.assertEqual(len(mock_search.call_args), 2) - self.assertEqual(mock_search.call_args[1]["index"], "INDEX") - self.assertEqual(body, res) - - @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") - @mock.patch.object(OpenSearch, "count") - @mock.patch.object(OpenSearch, "search") - def test_trends_filter__valid(self, mock_search, mock_count): - trends_params = { - "lens": "product", - "trend_interval": "year", - "sub_lens": "sub_product", - "trend_depth": 5, - "sub_lens_depth": 5, - "issue": "Incorrect information on your report", - } - body = load("trends_filter__valid") - mock_count.return_value = {"count": body["hits"]["total"]["value"]} - mock_search.return_value = body - - res = trends(**trends_params) - self.assertEqual(len(mock_search.call_args), 2) - self.assertEqual(mock_search.call_args[1]["index"], "INDEX") - self.assertEqual(body, res) - - @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") - @mock.patch.object(OpenSearch, "count") - @mock.patch.object(OpenSearch, "search") - def test_trends_top_self_filter__valid(self, mock_search, mock_count): - trends_params = { - "lens": "overview", - "trend_interval": "year", - "trend_depth": 5, - "issue": "Incorrect information on your report", - } - body = load("trends_filter__valid") - mock_count.return_value = {"count": body["hits"]["total"]["value"]} - mock_search.return_value = body - - res = trends(**trends_params) - self.assertEqual(len(mock_search.call_args), 2) - self.assertEqual(mock_search.call_args[1]["index"], "INDEX") - self.assertEqual(body, res) - self.assertTrue("company" not in res["aggregations"]) - - @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") - @mock.patch.object(OpenSearch, "count") - @mock.patch.object(OpenSearch, "search") - def test_trends_company_filter__valid(self, mock_search, mock_count): - default_exclude = AGG_EXCLUDE_FIELDS - trends_params = { - "lens": "overview", - "trend_interval": "year", - "company": "EQUIFAX, INC.", - } - - body = load("trends_company_filter__valid") - mock_count.return_value = {"count": body["hits"]["total"]["value"]} - mock_search.return_value = body - - res = trends(default_exclude, **trends_params) - self.assertEqual(len(mock_search.call_args), 2) - self.assertEqual(mock_search.call_args[1]["index"], "INDEX") - self.assertTrue("company" in res["aggregations"]) - - @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") - @mock.patch.object(OpenSearch, "count") - @mock.patch.object(OpenSearch, "search") - def test_trends_issue_focus__valid(self, mock_search, mock_count): - default_exclude = AGG_EXCLUDE_FIELDS - trends_params = { - "lens": "issue", - "trend_interval": "year", - "trend_depth": 5, - "focus": "Incorrect information on your report", - } - - body = load("trends_issue_focus__valid") - mock_count.return_value = {"count": body["hits"]["total"]["value"]} - mock_search.return_value = body - - res = trends(default_exclude, **trends_params) - self.assertEqual(len(mock_search.call_args), 2) - self.assertEqual(mock_search.call_args[1]["index"], "INDEX") - self.assertTrue("company" not in res["aggregations"]) - self.assertEqual( - len(res["aggregations"]["issue"]["issue"]["buckets"]), 1 - ) - - @mock.patch("complaint_search.es_interface._COMPLAINT_ES_INDEX", "INDEX") - @mock.patch.object(OpenSearch, "count") - @mock.patch.object(OpenSearch, "search") - def test_trends_issue_focus_company_filter__valid( - self, mock_search, mock_count - ): - default_exclude = AGG_EXCLUDE_FIELDS - trends_params = { - "lens": "issue", - "trend_interval": "year", - "trend_depth": 5, - "focus": "Incorrect information on your report", - "company": "EQUIFAX, INC.", - } - - body = load("trends_issue_focus_company_filter__valid") - mock_count.return_value = {"count": body["hits"]["total"]["value"]} - mock_search.return_value = body - - res = trends(default_exclude, **trends_params) - self.assertEqual(len(mock_search.call_args), 2) - self.assertEqual(mock_search.call_args[1]["index"], "INDEX") - self.assertTrue("company" in res["aggregations"]) - self.assertEqual( - len(res["aggregations"]["issue"]["issue"]["buckets"]), 1 - ) diff --git a/complaint_search/tests/test_serializer.py b/complaint_search/tests/test_serializer.py index 0810a8d7..b5dd98ea 100644 --- a/complaint_search/tests/test_serializer.py +++ b/complaint_search/tests/test_serializer.py @@ -4,10 +4,7 @@ from django.test import TestCase from complaint_search.defaults import PARAMS -from complaint_search.serializer import ( - SearchInputSerializer, - TrendsInputSerializer, -) +from complaint_search.serializer import SearchInputSerializer class SearchInputSerializerTests(TestCase): @@ -67,94 +64,3 @@ def test_is_valid__invalid_issue(self): '"issue\u2022subissue"' ], ) - - -class TrendsInputSerializerTests(TestCase): - def setUp(self): - self.data = {} - - def test_is_valid___no_required_params(self): - serializer = TrendsInputSerializer(data={}) - - self.assertFalse(serializer.is_valid()) - self.assertEqual( - serializer.errors["trend_interval"], ["This field is required."] - ) - - def test_is_valid__default_requred_params(self): - serializer = TrendsInputSerializer( - data={"trend_interval": "month", "lens": "overview"} - ) - - self.assertTrue(serializer.is_valid()) - - def test_is_valid__invalid_lens(self): - serializer = TrendsInputSerializer( - data={"trend_interval": "year", "lens": "foo"} - ) - - self.assertFalse(serializer.is_valid()) - self.assertEqual( - serializer.errors["lens"], ['"foo" is not a valid choice.'] - ) - - def test_is_valid__valid_lens(self): - serializer = TrendsInputSerializer( - data={"trend_interval": "year", "lens": "overview"} - ) - - self.assertTrue(serializer.is_valid()) - - def test_is_valid__no_sub_lens(self): - serializer = TrendsInputSerializer( - data={"trend_interval": "year", "lens": "product"} - ) - - self.assertFalse(serializer.is_valid()) - self.assertEqual( - serializer.errors["non_field_errors"], - [ - "Either Focus or Sub-lens is required for lens 'product'." - " Valid sub-lens are: ('sub_product', 'issue', 'company', 'tags')" - ], - ) - - def test_is_valid__invalid_sub_lens(self): - serializer = TrendsInputSerializer( - data={ - "trend_interval": "year", - "lens": "product", - "sub_lens": "foobar", - } - ) - - self.assertFalse(serializer.is_valid()) - self.assertEqual( - serializer.errors["non_field_errors"], - [ - "'foobar' is not a valid sub-lens for 'product'. Valid " - "sub-lens are: ('sub_product', 'issue', 'company', 'tags')" - ], - ) - - def test_is_valid__valid_sub_lens(self): - serializer = TrendsInputSerializer( - data={ - "trend_interval": "year", - "lens": "product", - "sub_lens": "company", - } - ) - - self.assertTrue(serializer.is_valid()) - - def test_is_valid__valid_focus(self): - serializer = TrendsInputSerializer( - data={ - "trend_interval": "year", - "lens": "product", - "focus": "Debt collection", - } - ) - - self.assertTrue(serializer.is_valid()) diff --git a/complaint_search/tests/test_views_search.py b/complaint_search/tests/test_views_search.py index 80ac6985..c41d90eb 100644 --- a/complaint_search/tests/test_views_search.py +++ b/complaint_search/tests/test_views_search.py @@ -644,20 +644,6 @@ def test_search_with_company_public_response__valid(self, mock_essearch): ) self.assertEqual("OK", response.data) - @mock.patch("complaint_search.es_interface.search") - def test_search_with_has_narrative__valid(self, mock_essearch): - url = reverse("complaint_search:search") - url += "?has_narrative=Yes&has_narrative=No" - mock_essearch.return_value = "OK" - response = self.client.get(url) - self.assertEqual(status.HTTP_200_OK, response.status_code) - # -*- coding: utf-8 -*- - mock_essearch.assert_called_once_with( - agg_exclude=AGG_EXCLUDE_FIELDS, - **self.buildDefaultParams({"has_narrative": ["Yes", "No"]}), - ) - self.assertEqual("OK", response.data) - @mock.patch("complaint_search.es_interface.search") def test_search_with_submitted_via__valid(self, mock_essearch): url = reverse("complaint_search:search") diff --git a/complaint_search/tests/test_views_states.py b/complaint_search/tests/test_views_states.py deleted file mode 100644 index 48ad34dc..00000000 --- a/complaint_search/tests/test_views_states.py +++ /dev/null @@ -1,84 +0,0 @@ -import copy -from unittest import mock - -from parameterized import parameterized -from rest_framework import status -from rest_framework.test import APITestCase - -from complaint_search.defaults import AGG_EXCLUDE_FIELDS, PARAMS -from complaint_search.serializer import SearchInputSerializer - - -try: - from django.urls import reverse - -except ImportError: - from django.core.urlresolvers import reverse - - -class StatesTests(APITestCase): - def setUp(self): - pass - - def buildDefaultParams(self, overrides): - params = copy.deepcopy(PARAMS) - del params["search_after"] - params.update(overrides) - return params - - @mock.patch("complaint_search.es_interface.states_agg") - def test_states_no_param(self, mock_essearch): - """ - Searching with no parameters - """ - url = reverse("complaint_search:states") - mock_essearch.return_value = "OK" - response = self.client.get(url) - self.assertEqual(response.status_code, status.HTTP_200_OK) - mock_essearch.assert_called_once_with( - agg_exclude=AGG_EXCLUDE_FIELDS, **self.buildDefaultParams({}) - ) - self.assertEqual("OK", response.data) - - @parameterized.expand( - [ - [("complaint_what_happened", "complaint_what_happened field")], - [("company", "company field")], - [("all", "all fields")], - ] - ) - @mock.patch("complaint_search.es_interface.states_agg") - def test_states_with_field__valid(self, test_pair, mock_essearch): - url = reverse("complaint_search:states") - params = {"field": test_pair[0]} - mock_essearch.return_value = "OK" - response = self.client.get(url, params) - self.assertEqual(status.HTTP_200_OK, response.status_code) - self.assertEqual("OK", response.data) - - calls = [ - mock.call( - agg_exclude=AGG_EXCLUDE_FIELDS, - **self.buildDefaultParams( - { - "field": SearchInputSerializer.FIELD_MAP.get( - test_pair[0], test_pair[0] - ) - } - ), - ) - ] - mock_essearch.assert_has_calls(calls) - - @mock.patch("complaint_search.es_interface.states_agg") - def test_state_with_field__invalid(self, mock_essearch): - url = reverse("complaint_search:states") - params = {"field": "invalid_choice"} - mock_essearch.return_value = "OK" - response = self.client.get(url, params) - self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code) - mock_essearch.assert_not_called() - self.assertDictEqual( - {"field": ['"invalid_choice" is not a valid choice.']}, - response.data, - ) diff --git a/complaint_search/tests/test_views_trends.py b/complaint_search/tests/test_views_trends.py deleted file mode 100644 index 2a8be075..00000000 --- a/complaint_search/tests/test_views_trends.py +++ /dev/null @@ -1,69 +0,0 @@ -import copy -from unittest import mock - -from rest_framework import status -from rest_framework.test import APITestCase - -from complaint_search.defaults import PARAMS - - -try: - from django.urls import reverse -except ImportError: - from django.core.urlresolvers import reverse - - -class TrendsTests(APITestCase): - def setUp(self): - pass - - def buildDefaultParams(self, overrides): - params = copy.deepcopy(PARAMS) - params.update(overrides) - return params - - @mock.patch("complaint_search.es_interface.trends") - def test_trends_no_required_params__fails(self, mock_essearch): - """ - Searching with no parameters - """ - url = reverse("complaint_search:trends") - response = self.client.get(url) - self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - mock_essearch.assert_not_called() - - @mock.patch("complaint_search.es_interface.trends") - def test_trends_default_params__fails(self, mock_essearch): - """ - Searching with no required trends parameters - """ - url = reverse("complaint_search:trends") - response = self.client.get(url, **self.buildDefaultParams({})) - self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - mock_essearch.assert_not_called() - - @mock.patch("complaint_search.es_interface.trends") - def test_trends_invalid_params__fails(self, mock_essearch): - """ - Searching with invalid required trends parameters - """ - url = reverse("complaint_search:trends") - params = {"lens": "foo"} - response = self.client.get(url, params) - self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - mock_essearch.assert_not_called() - self.assertTrue("lens" in response.data.keys()) - self.assertTrue("trend_interval" in response.data.keys()) - - @mock.patch("complaint_search.es_interface.trends") - def test_trends_default_params__passes(self, mock_essearch): - """ - Searching with default but valid required trends parameters - """ - url = reverse("complaint_search:trends") - params = {"lens": "overview", "trend_interval": "month"} - - mock_essearch.return_value = "results" - response = self.client.get(url, params) - self.assertEqual(response.status_code, status.HTTP_200_OK) - mock_essearch.assert_called_once() diff --git a/complaint_search/urls.py b/complaint_search/urls.py index b25cba34..8a85185b 100644 --- a/complaint_search/urls.py +++ b/complaint_search/urls.py @@ -1,5 +1,4 @@ from django.urls import re_path -from django.views.generic.base import RedirectView import complaint_search.views @@ -21,7 +20,4 @@ r"^(?P[0-9]+)$", complaint_search.views.document, name="complaint" ), re_path(r"^$", complaint_search.views.search, name="search"), - re_path(r"^geo/states$", complaint_search.views.states, name="states"), - re_path(r"^geo", RedirectView.as_view(url="/geo/states"), name="geo"), - re_path(r"^trends$", complaint_search.views.trends, name="trends"), ] diff --git a/complaint_search/views.py b/complaint_search/views.py index b7b1cdce..ef864fac 100644 --- a/complaint_search/views.py +++ b/complaint_search/views.py @@ -23,7 +23,6 @@ from complaint_search.serializer import ( SearchInputSerializer, SuggestFilterInputSerializer, - TrendsInputSerializer, ) from complaint_search.throttling import ( DocumentAnonRateThrottle, @@ -45,9 +44,7 @@ "date_received_max", "date_received_min", "field", - "focus", "frm", - "lens", "no_aggs", "no_highlight", "page", @@ -55,10 +52,6 @@ "search_term", "size", "sort", - "sub_lens", - "sub_lens_depth", - "trend_depth", - "trend_interval", ) @@ -233,45 +226,3 @@ def removekey(d, key): def document(request, id): results = es_interface.document(id) return Response(results, headers=_build_headers()) - - -# ----------------------------------------------------------------------------- -# Request Handlers: Geo - - -@api_view(["GET"]) -@catch_es_error -def states(request): - data = _parse_query_params(request.query_params) - serializer = SearchInputSerializer(data=data) - - if not serializer.is_valid(): - return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) - - results = es_interface.states_agg( - agg_exclude=AGG_EXCLUDE_FIELDS, **serializer.validated_data - ) - headers = _build_headers() - - return Response(results, headers=headers) - - -# ----------------------------------------------------------------------------- -# Request Handlers: Trends - - -@api_view(["GET"]) -@catch_es_error -def trends(request): - data = _parse_query_params(request.query_params) - serializer = TrendsInputSerializer(data=data) - - if not serializer.is_valid(): - return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) - - results = es_interface.trends( - agg_exclude=AGG_EXCLUDE_FIELDS, **serializer.validated_data - ) - headers = _build_headers() - - return Response(results, headers=headers) diff --git a/setup.py b/setup.py index b63c0505..82aa3a74 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,6 @@ def get_git_version(): "requests>=2.32.4,<3", "opensearch-py>=2.1.0,<=3.0.0", "django-localflavor>=4.0,<5.0", - "django-flags>=5", ] testing_extras = [ diff --git a/swagger-config.yaml b/swagger-config.yaml deleted file mode 100644 index 544987c0..00000000 --- a/swagger-config.yaml +++ /dev/null @@ -1,787 +0,0 @@ -openapi: 3.0.0 -info: - version: 1.0.0 - title: Consumer Complaint Database API - description: The API for searching the Consumer Complaint Database - termsOfService: 'https://cfpb.github.io/source-code-policy/' - contact: - name: Report API Issues - url: https://github.com/cfpb/ccdb5-api/issues - license: - name: Creative Commons License CC0 - url: 'https://github.com/cfpb/ccdb5-api/blob/main/LICENSE' -paths: - /: - get: - tags: - - Complaints - summary: Search consumer complaints - description: Search the contents of the consumer complaint database - parameters: - - $ref: '#/components/parameters/search_term' - - $ref: '#/components/parameters/field' - - $ref: '#/components/parameters/from' - - $ref: '#/components/parameters/size' - - $ref: '#/components/parameters/sort' - - $ref: '#/components/parameters/format' - - $ref: '#/components/parameters/no_aggs' - - $ref: '#/components/parameters/no_highlight' - - $ref: '#/components/parameters/company' - - $ref: '#/components/parameters/company_public_response' - - $ref: '#/components/parameters/company_received_max' - - $ref: '#/components/parameters/company_received_min' - - $ref: '#/components/parameters/company_response' - - $ref: '#/components/parameters/date_received_max' - - $ref: '#/components/parameters/date_received_min' - - $ref: '#/components/parameters/has_narrative' - - $ref: '#/components/parameters/issue' - - $ref: '#/components/parameters/product' - - $ref: '#/components/parameters/search_after' - - $ref: '#/components/parameters/state' - - $ref: '#/components/parameters/submitted_via' - - $ref: '#/components/parameters/tags' - - $ref: '#/components/parameters/timely' - - $ref: '#/components/parameters/zip_code' - responses: - '200': - description: successful operation - content: - application/json: - schema: - $ref: '#/components/schemas/SearchResult' - text/csv: - schema: - $ref: '#/components/schemas/SearchResult' - '400': - description: Invalid status value - /_suggest: - get: - tags: - - Typeahead - summary: Suggest possible searches - description: The endpoint for the main search box in the UI - parameters: - - $ref: '#/components/parameters/size' - - $ref: '#/components/parameters/suggest_text' - responses: - '200': - description: successful operation - content: - application/json: - schema: - $ref: '#/components/schemas/SuggestResult' - '400': - description: Invalid input - /_suggest_company: - get: - tags: - - Typeahead - summary: Suggest possible companies - description: Provide a list of companies that match the input text - parameters: - - $ref: '#/components/parameters/suggest_text' - - $ref: '#/components/parameters/size' - - $ref: '#/components/parameters/company_public_response' - - $ref: '#/components/parameters/company_received_max' - - $ref: '#/components/parameters/company_received_min' - - $ref: '#/components/parameters/company_response' - - $ref: '#/components/parameters/date_received_max' - - $ref: '#/components/parameters/date_received_min' - - $ref: '#/components/parameters/has_narrative' - - $ref: '#/components/parameters/issue' - - $ref: '#/components/parameters/product' - - $ref: '#/components/parameters/state' - - $ref: '#/components/parameters/submitted_via' - - $ref: '#/components/parameters/tags' - - $ref: '#/components/parameters/timely' - - $ref: '#/components/parameters/zip_code' - responses: - '200': - description: successful operation - content: - application/json: - schema: - $ref: '#/components/schemas/SuggestResult' - '400': - description: Invalid input - /_suggest_zip: - get: - tags: - - Typeahead - summary: Suggest possible zip codes - description: Provide a list of zip codes that match the input text - parameters: - - $ref: '#/components/parameters/suggest_text' - - $ref: '#/components/parameters/size' - - $ref: '#/components/parameters/company_public_response' - - $ref: '#/components/parameters/company_received_max' - - $ref: '#/components/parameters/company_received_min' - - $ref: '#/components/parameters/company_response' - - $ref: '#/components/parameters/date_received_max' - - $ref: '#/components/parameters/date_received_min' - - $ref: '#/components/parameters/has_narrative' - - $ref: '#/components/parameters/issue' - - $ref: '#/components/parameters/product' - - $ref: '#/components/parameters/state' - - $ref: '#/components/parameters/submitted_via' - - $ref: '#/components/parameters/tags' - - $ref: '#/components/parameters/timely' - - $ref: '#/components/parameters/zip_code' - responses: - '200': - description: successful operation - content: - application/json: - schema: - $ref: '#/components/schemas/SuggestResult' - '400': - description: Invalid input - '/{complaintId}': - get: - tags: - - Complaints - summary: Find consumer complaint by ID - description: Get complaint details for a specific ID - parameters: - - name: complaintId - in: path - description: ID of the complaint - required: true - schema: - type: integer - format: int64 - minimum: 0 - maximum: 9999999999 - responses: - '200': - description: successful operation - content: - application/json: - schema: - $ref: '#/components/schemas/Complaint' - '400': - description: Invalid ID supplied - '404': - description: Complaint not found - /geo/states: - get: - tags: - - Complaints - summary: Get the state-by-state information - description: Get complaint information broken down by states - parameters: - - $ref: '#/components/parameters/search_term' - - $ref: '#/components/parameters/field' - - $ref: '#/components/parameters/company' - - $ref: '#/components/parameters/company_public_response' - - $ref: '#/components/parameters/company_received_max' - - $ref: '#/components/parameters/company_received_min' - - $ref: '#/components/parameters/company_response' - - $ref: '#/components/parameters/date_received_max' - - $ref: '#/components/parameters/date_received_min' - - $ref: '#/components/parameters/has_narrative' - - $ref: '#/components/parameters/issue' - - $ref: '#/components/parameters/product' - - $ref: '#/components/parameters/state' - - $ref: '#/components/parameters/submitted_via' - - $ref: '#/components/parameters/tags' - - $ref: '#/components/parameters/timely' - - $ref: '#/components/parameters/zip_code' - responses: - '200': - description: successful operation - content: - application/json: - schema: - $ref: '#/components/schemas/StatesResult' - /trends: - get: - tags: - - Trends - summary: "List trends" - description: "Return specific aggregations for a search" - parameters: - - $ref: '#/components/parameters/search_term' - - $ref: '#/components/parameters/field' - - $ref: '#/components/parameters/company' - - $ref: '#/components/parameters/company_public_response' - - $ref: '#/components/parameters/company_received_max' - - $ref: '#/components/parameters/company_received_min' - - $ref: '#/components/parameters/company_response' - - $ref: '#/components/parameters/date_received_max' - - $ref: '#/components/parameters/date_received_min' - - $ref: '#/components/parameters/focus' - - $ref: '#/components/parameters/has_narrative' - - $ref: '#/components/parameters/issue' - - $ref: '#/components/parameters/lens' - - $ref: '#/components/parameters/product' - - $ref: '#/components/parameters/state' - - $ref: '#/components/parameters/submitted_via' - - $ref: '#/components/parameters/sub_lens' - - $ref: '#/components/parameters/sub_lens_depth' - - $ref: '#/components/parameters/tags' - - $ref: '#/components/parameters/timely' - - $ref: '#/components/parameters/trend_depth' - - $ref: '#/components/parameters/trend_interval' - - $ref: '#/components/parameters/zip_code' - responses: - '200': - description: successful operation - content: - application/json: - schema: - $ref: '#/components/schemas/TrendsResult' - text/csv: - schema: - $ref: '#/components/schemas/TrendsResult' - '400': - description: Invalid status value -tags: - - name: Complaints - description: These endpoints provide access to consumer complaints - - name: Trends - description: These endpoints provide access aggregated consumer complaint data - - name: Typeahead - description: These endpoints support the typeahead boxes in the UI -externalDocs: - description: Additional API Information - url: 'https://cfpb.github.io/api/ccdb/' -servers: - - url: 'https://www.consumerfinance.gov/data-research/consumer-complaints/search/api/v1/' -components: - parameters: - company: - name: company - in: query - description: Filter the results to only return these companies - explode: true - schema: - type: array - items: - type: string - company_public_response: - name: company_public_response - in: query - description: Filter the results to only return these types of public response by the company - explode: true - schema: - type: array - items: - type: string - company_received_max: - name: company_received_max - in: query - description: Return results with date < company_received_max (i.e. 2017-03-04) - schema: - type: string - format: date - company_received_min: - name: company_received_min - in: query - description: Return results with date >= company_received_min (i.e. 2017-03-04) - schema: - type: string - format: date - company_response: - name: company_response - in: query - description: Filter the results to only return these types of response by the company - explode: true - schema: - type: array - items: - type: string - date_received_max: - name: date_received_max - in: query - description: Return results with date < date_received_max (i.e. 2017-03-04) - schema: - type: string - format: date - date_received_min: - name: date_received_min - in: query - description: Return results with date >= date_received_min (i.e. 2017-03-04) - schema: - type: string - format: date - field: - name: field - in: query - description: If the parameter "search_term" has a value, use "field" to specify which field is searched. If not specified, "complaint_what_happened" will be searched. - schema: - type: string - enum: - - complaint_what_happened - - company_public_response - - all - default: complaint_what_happened - focus: - name: focus - in: query - description: The name of the product or company on which to focus charts for products and issues - explode: true - schema: - type: array - items: - type: string - format: - name: format - in: query - description: Format to be returned, if this parameter is not specified, frm/size parameters can be used properly, but if a format is specified for exporting, frm/size will be ignored - schema: - type: string - enum: - - csv - default: csv - from: - name: frm - in: query - description: Return results starting from a specific index, only if format parameter is not specified, ignore otherwise - schema: - type: integer - format: int64 - minimum: 1 - maximum: 100000 - default: 0 - has_narrative: - name: has_narrative - in: query - description: Filter the results to only return the specified state of whether it has narrative in the complaint or not, i.e. yes, no - explode: true - schema: - type: array - items: - type: string - issue: - name: issue - in: query - description: 'Filter the results to only return these types of issue and subissue, i.e. product-only: Getting a Loan, subproduct needs to include product, separated by ''•'', Getting a Loan•Can''t qualify for a loan' - explode: true - schema: - type: array - items: - type: string - lens: - name: lens - in: query - required: true - description: The data lens through which to view complaint trends over time. - schema: - type: string - enum: - - overview - - issue - - product - - tags - default: overview - no_aggs: - name: no_aggs - in: query - description: Include aggregations in result or not, True means no aggregations will be included, False means aggregations will be included. - schema: - type: boolean - default: false - no_highlight: - name: no_highlight - in: query - description: Include highlight of search term in result or not, True means no highlighting will be included, False means highlighting will be included. - schema: - type: boolean - default: false - product: - name: product - in: query - description: 'Filter the results to only return these types of product and subproduct, i.e. product-only: Mortgage, subproduct needs to include product, separated by ''•'', Mortgage•FHA mortgage' - explode: true - schema: - type: array - items: - type: string - search_after: - name: search_after - in: query - description: Used in conjunction with frm parameter to paginate results. This value is calculated by combining the values from the break_points object in the _meta key from the api response. For instance to paginate to the nth page, use the value from the break_points n:[k, id] to pass to the API the value page=n&frm=25&search_after=k_id Please see https://github.com/cfpb/cfpb.github.io/issues/292 for more detailed examples - schema: - type: string - search_term: - name: search_term - in: query - description: Return results containing specific term - schema: - type: string - size: - name: size - in: query - description: Limit the size of the results - schema: - type: integer - format: int64 - minimum: 1 - maximum: 100 - default: 10 - sort: - name: sort - in: query - description: Return results sort in a particular order - schema: - type: string - enum: - - relevance_desc - - relevance_asc - - created_date_desc - - created_date_asc - default: relevance_desc - state: - name: state - in: query - description: Filter the results to only return these states (use abbreviation, i.e. CA, VA) - explode: true - schema: - type: array - items: - type: string - sub_lens: - name: sub_lens - in: query - description: The sub-lens through which to view complaint trends over time. - schema: - type: string - enum: - - issue - - product - - sub_product - - sub_issue - - tags - sub_lens_depth: - name: sub_lens_depth - in: query - description: The top X trend sub aggregations will be returned, where X is the supplied sub_lens_depth. - schema: - type: integer - maximum: 10000000 - minimum: 5 - format: int64 - default: 10 - submitted_via: - name: submitted_via - in: query - description: Filter the results to only return these types of way consumers submitted their complaints - explode: true - schema: - type: array - items: - type: string - suggest_text: - name: text - in: query - description: text to use for suggestions - required: true - schema: - type: string - tags: - name: tags - in: query - description: Filter the results to only return these types of tag - explode: true - schema: - type: array - items: - type: string - timely: - name: timely - in: query - description: Filter the results to show whether a response was timely - explode: true - schema: - type: array - items: - type: string - trend_depth: - name: trend_depth - in: query - description: The top X trend aggregations will be returned, where X is the supplied trend_depth. - schema: - type: integer - maximum: 10000000 - minimum: 5 - format: int64 - default: 10 - trend_interval: - name: trend_interval - in: query - description: The interval of time to use for trends aggregations histograms. When using day intervals, we recommend querying for date_received_min / max periods of less than one year. - required: true - schema: - type: string - enum: - - year - - quarter - - month - - week - - day - zip_code: - name: zip_code - in: query - description: Filter the results to only return these zip codes - explode: true - schema: - type: array - items: - type: string - schemas: - Aggregation: - type: object - description: An OpenSearch aggregation - properties: - doc_count: - type: integer - description: The total number of complaints covered in this aggregation - field: - type: object - description: The name of the ` being aggregated - properties: - buckets: - type: array - items: - $ref: '#/components/schemas/Bucket' - doc_count_error_upper_bound: - type: integer - description: The number of possible errors that occurred when searching the shards - sum_other_doc_count: - type: integer - description: The number of complaints that were not included in this aggregation. - AggregationDate: - type: object - properties: - value_as_string: - type: string - format: date-time - description: ISO-8601 formatted date (yyyy-mm-ddTHH:MM:SSZZ) - value: - type: number - description: Seconds since 1970 (Unix Epoch) - Bucket: - type: object - properties: - doc_count: - type: integer - description: The number of complaints that match this key - key: - type: string - Complaint: - type: object - externalDocs: - description: Official documentation - url: 'https://cfpb.github.io/api/ccdb/fields.html' - properties: - company: - type: string - description: The complaint is about this company - company_public_response: - type: string - description: 'The company''s optional, public-facing response to a consumer''s complaint' - company_response: - type: string - description: The response from the company about this complaint - complaint_id: - type: integer - description: The unique identification number for a complaint - complaint_what_happened: - type: string - description: A description of the complaint provided by the consumer - date_received: - type: string - format: date - description: The date the CFPB received the complaint - date_sent_to_company: - type: string - description: The date the CFPB sent the complaint to the company - has_narrative: - type: boolean - description: Indicates this complaint has a narrative - issue: - type: string - description: The issue the consumer identified in the complaint - product: - type: string - description: The type of product the consumer identified in the complaint - state: - type: string - description: The state of the mailing address provided by the consumer - sub_issue: - type: string - description: The sub-issue the consumer identified in the complaint - sub_product: - type: string - description: The type of sub-product the consumer identified in the complaint - submitted_via: - type: string - description: How the complaint was submitted to the CFPB - tags: - type: string - description: Data that supports easier searching and sorting of complaints - timely: - type: string - description: Indicates whether the company gave a timely response or not - zip_code: - type: string - description: The mailing ZIP code provided by the consumer - Hit: - type: object - description: A single OpenSearch result - properties: - _source: - $ref: '#/components/schemas/Complaint' - Hits: - type: object - description: A set of complaints that matched the query - properties: - hits: - type: array - items: - $ref: '#/components/schemas/Hit' - max_score: - type: number - description: The highest score in the results - format: float - total: - type: object - properties: - value: - type: integer - description: "The count of matching hits, accurate in our code even above 10,000 hits" - relation: - type: string - description: "Indicates the accuracy of the default ES response, whether the value is accurate (eq) or a lower bound (gte)" - Meta: - type: object - properties: - break_points: - type: object - description: Contains key value pairs of page and arrays. Used to paginate OpenSearch results in list view - has_data_issue: - type: boolean - description: Indicates there has been an issue with the most recent data load - is_data_stale: - type: boolean - description: Indicates the most recent data is over 5 business days old - is_narrative_stale: - type: boolean - description: Indicates the most recent narratives are over 5 business days old - last_indexed: - type: string - description: The timestamp of the most recently indexed complaint - format: dateTime - last_updated: - type: string - description: The timestamp of the most recent complaint - format: dateTime - license: - type: string - description: The open source license under which the API operates - total_record_count: - type: integer - description: The total number of complaints currently indexed - MultiLevelAggregation: - properties: - doc_count: - type: integer - description: The total number of complaints covered in this aggregation - field: - type: object - description: The name of the field being aggregated - properties: - buckets: - type: array - items: - $ref: '#/components/schemas/MultiLevelBucket' - doc_count_error_upper_bound: - type: integer - description: The number of possible errors that occurred when searching the shards - sum_other_doc_count: - type: integer - description: The number of complaints that were not included in this aggregation. - MultiLevelBucket: - type: object - properties: - doc_count: - type: integer - description: The number of complaints that match this key - field.raw: - type: object - description: The next level of aggregations - properties: - buckets: - type: array - items: - $ref: '#/components/schemas/Aggregation' - key: - type: string - SearchResult: - type: object - properties: - _meta: - $ref: '#/components/schemas/Meta' - aggregations: - type: object - properties: - company_public_response: - $ref: '#/components/schemas/Aggregation' - company_response: - $ref: '#/components/schemas/Aggregation' - has_narrative: - $ref: '#/components/schemas/Aggregation' - issue: - $ref: '#/components/schemas/MultiLevelAggregation' - product: - $ref: '#/components/schemas/MultiLevelAggregation' - state: - $ref: '#/components/schemas/Aggregation' - submitted_via: - $ref: '#/components/schemas/Aggregation' - tags: - $ref: '#/components/schemas/Aggregation' - timely: - $ref: '#/components/schemas/Aggregation' - zip_code: - $ref: '#/components/schemas/Aggregation' - hits: - $ref: '#/components/schemas/Hits' - StatesResult: - type: object - properties: - aggregations: - type: object - properties: - issue: - $ref: '#/components/schemas/MultiLevelAggregation' - product: - $ref: '#/components/schemas/MultiLevelAggregation' - state: - $ref: '#/components/schemas/MultiLevelAggregation' - SuggestResult: - type: array - items: - type: string - TrendsResult: - type: object - properties: - aggregations: - type: object - properties: - company: - $ref: '#/components/schemas/MultiLevelAggregation' - issue: - $ref: '#/components/schemas/MultiLevelAggregation' - product: - $ref: '#/components/schemas/MultiLevelAggregation' - sub_issue: - $ref: '#/components/schemas/MultiLevelAggregation' - sub_product: - $ref: '#/components/schemas/MultiLevelAggregation' - tags: - $ref: '#/components/schemas/MultiLevelAggregation' - links: {} - callbacks: {} -security: [] From 985c05508e74c6fa0193d98d026a554cf56ef207 Mon Sep 17 00:00:00 2001 From: Wyatt Pearsall Date: Mon, 17 Aug 2026 14:03:56 -0700 Subject: [PATCH 2/6] restore flags --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 82aa3a74..b63c0505 100644 --- a/setup.py +++ b/setup.py @@ -53,6 +53,7 @@ def get_git_version(): "requests>=2.32.4,<3", "opensearch-py>=2.1.0,<=3.0.0", "django-localflavor>=4.0,<5.0", + "django-flags>=5", ] testing_extras = [ From f66d6af04076727096e32792bcdaac9737c231bf Mon Sep 17 00:00:00 2001 From: Wyatt Pearsall Date: Tue, 18 Aug 2026 09:24:11 -0700 Subject: [PATCH 3/6] fixup api responses --- complaint_search/defaults.py | 5 +- complaint_search/es_builders.py | 6 +- complaint_search/serializer.py | 5 - complaint_search/views.py | 1 - swagger-config.yaml | 787 ++++++++++++++++++++++++++++++++ 5 files changed, 789 insertions(+), 15 deletions(-) create mode 100644 swagger-config.yaml diff --git a/complaint_search/defaults.py b/complaint_search/defaults.py index 23b731be..acf52e22 100644 --- a/complaint_search/defaults.py +++ b/complaint_search/defaults.py @@ -24,8 +24,8 @@ RESULT_SIZE_OPTIONS = [10, 50, 100] PARAMS = { + "field": "all", "format": "default", - "field": "complaint_what_happened", "frm": 0, "search_after": "", "page": 1, @@ -43,10 +43,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -69,7 +67,6 @@ ("sub_product", "Sub-product"), ("issue", "Issue"), ("sub_issue", "Sub-issue"), - ("complaint_what_happened", "Consumer complaint narrative"), ("company_public_response", "Company public response"), ("company", "Company"), ("state", "State"), diff --git a/complaint_search/es_builders.py b/complaint_search/es_builders.py index 63a0eea8..3b501700 100644 --- a/complaint_search/es_builders.py +++ b/complaint_search/es_builders.py @@ -265,10 +265,7 @@ def _build_sort(self): return [{sort_field: {"order": sort_order}}, {"_id": sort_order}] def _build_source(self): - source = list(SOURCE_FIELDS) - if self.params.get("format") in EXPORT_FORMATS: - source.remove("has_narrative") - return source + return list(SOURCE_FIELDS) def build(self): search = { @@ -312,7 +309,6 @@ class AggregationBuilder(BaseBuilder): "company", "company_public_response", "company_response", - "has_narrative", "issue", "product", "state", diff --git a/complaint_search/serializer.py b/complaint_search/serializer.py index e9330d12..601847c6 100644 --- a/complaint_search/serializer.py +++ b/complaint_search/serializer.py @@ -16,13 +16,11 @@ class SearchInputSerializer(serializers.Serializer): ) # Field Choices - FIELD_NARRATIVE = "complaint_what_happened" FIELD_COMPANY = "company" FIELD_ALL = "all" FIELD_ALL_ES = "_all" FIELD_CHOICES = ( - (FIELD_NARRATIVE, "complaint_what_happened field"), (FIELD_COMPANY, "company field"), (FIELD_ALL, "all fields"), (FIELD_ALL_ES, "all fields"), @@ -85,9 +83,6 @@ class SearchInputSerializer(serializers.Serializer): company_public_response = serializers.ListField( child=serializers.CharField(max_length=200), required=False ) - has_narrative = serializers.ListField( - child=serializers.CharField(max_length=200), required=False - ) submitted_via = serializers.ListField( child=serializers.CharField(max_length=200), required=False ) diff --git a/complaint_search/views.py b/complaint_search/views.py index ef864fac..04f27c40 100644 --- a/complaint_search/views.py +++ b/complaint_search/views.py @@ -59,7 +59,6 @@ "company", "company_public_response", "company_response", - "has_narrative", "issue", "product", "state", diff --git a/swagger-config.yaml b/swagger-config.yaml new file mode 100644 index 00000000..544987c0 --- /dev/null +++ b/swagger-config.yaml @@ -0,0 +1,787 @@ +openapi: 3.0.0 +info: + version: 1.0.0 + title: Consumer Complaint Database API + description: The API for searching the Consumer Complaint Database + termsOfService: 'https://cfpb.github.io/source-code-policy/' + contact: + name: Report API Issues + url: https://github.com/cfpb/ccdb5-api/issues + license: + name: Creative Commons License CC0 + url: 'https://github.com/cfpb/ccdb5-api/blob/main/LICENSE' +paths: + /: + get: + tags: + - Complaints + summary: Search consumer complaints + description: Search the contents of the consumer complaint database + parameters: + - $ref: '#/components/parameters/search_term' + - $ref: '#/components/parameters/field' + - $ref: '#/components/parameters/from' + - $ref: '#/components/parameters/size' + - $ref: '#/components/parameters/sort' + - $ref: '#/components/parameters/format' + - $ref: '#/components/parameters/no_aggs' + - $ref: '#/components/parameters/no_highlight' + - $ref: '#/components/parameters/company' + - $ref: '#/components/parameters/company_public_response' + - $ref: '#/components/parameters/company_received_max' + - $ref: '#/components/parameters/company_received_min' + - $ref: '#/components/parameters/company_response' + - $ref: '#/components/parameters/date_received_max' + - $ref: '#/components/parameters/date_received_min' + - $ref: '#/components/parameters/has_narrative' + - $ref: '#/components/parameters/issue' + - $ref: '#/components/parameters/product' + - $ref: '#/components/parameters/search_after' + - $ref: '#/components/parameters/state' + - $ref: '#/components/parameters/submitted_via' + - $ref: '#/components/parameters/tags' + - $ref: '#/components/parameters/timely' + - $ref: '#/components/parameters/zip_code' + responses: + '200': + description: successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/SearchResult' + text/csv: + schema: + $ref: '#/components/schemas/SearchResult' + '400': + description: Invalid status value + /_suggest: + get: + tags: + - Typeahead + summary: Suggest possible searches + description: The endpoint for the main search box in the UI + parameters: + - $ref: '#/components/parameters/size' + - $ref: '#/components/parameters/suggest_text' + responses: + '200': + description: successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/SuggestResult' + '400': + description: Invalid input + /_suggest_company: + get: + tags: + - Typeahead + summary: Suggest possible companies + description: Provide a list of companies that match the input text + parameters: + - $ref: '#/components/parameters/suggest_text' + - $ref: '#/components/parameters/size' + - $ref: '#/components/parameters/company_public_response' + - $ref: '#/components/parameters/company_received_max' + - $ref: '#/components/parameters/company_received_min' + - $ref: '#/components/parameters/company_response' + - $ref: '#/components/parameters/date_received_max' + - $ref: '#/components/parameters/date_received_min' + - $ref: '#/components/parameters/has_narrative' + - $ref: '#/components/parameters/issue' + - $ref: '#/components/parameters/product' + - $ref: '#/components/parameters/state' + - $ref: '#/components/parameters/submitted_via' + - $ref: '#/components/parameters/tags' + - $ref: '#/components/parameters/timely' + - $ref: '#/components/parameters/zip_code' + responses: + '200': + description: successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/SuggestResult' + '400': + description: Invalid input + /_suggest_zip: + get: + tags: + - Typeahead + summary: Suggest possible zip codes + description: Provide a list of zip codes that match the input text + parameters: + - $ref: '#/components/parameters/suggest_text' + - $ref: '#/components/parameters/size' + - $ref: '#/components/parameters/company_public_response' + - $ref: '#/components/parameters/company_received_max' + - $ref: '#/components/parameters/company_received_min' + - $ref: '#/components/parameters/company_response' + - $ref: '#/components/parameters/date_received_max' + - $ref: '#/components/parameters/date_received_min' + - $ref: '#/components/parameters/has_narrative' + - $ref: '#/components/parameters/issue' + - $ref: '#/components/parameters/product' + - $ref: '#/components/parameters/state' + - $ref: '#/components/parameters/submitted_via' + - $ref: '#/components/parameters/tags' + - $ref: '#/components/parameters/timely' + - $ref: '#/components/parameters/zip_code' + responses: + '200': + description: successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/SuggestResult' + '400': + description: Invalid input + '/{complaintId}': + get: + tags: + - Complaints + summary: Find consumer complaint by ID + description: Get complaint details for a specific ID + parameters: + - name: complaintId + in: path + description: ID of the complaint + required: true + schema: + type: integer + format: int64 + minimum: 0 + maximum: 9999999999 + responses: + '200': + description: successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/Complaint' + '400': + description: Invalid ID supplied + '404': + description: Complaint not found + /geo/states: + get: + tags: + - Complaints + summary: Get the state-by-state information + description: Get complaint information broken down by states + parameters: + - $ref: '#/components/parameters/search_term' + - $ref: '#/components/parameters/field' + - $ref: '#/components/parameters/company' + - $ref: '#/components/parameters/company_public_response' + - $ref: '#/components/parameters/company_received_max' + - $ref: '#/components/parameters/company_received_min' + - $ref: '#/components/parameters/company_response' + - $ref: '#/components/parameters/date_received_max' + - $ref: '#/components/parameters/date_received_min' + - $ref: '#/components/parameters/has_narrative' + - $ref: '#/components/parameters/issue' + - $ref: '#/components/parameters/product' + - $ref: '#/components/parameters/state' + - $ref: '#/components/parameters/submitted_via' + - $ref: '#/components/parameters/tags' + - $ref: '#/components/parameters/timely' + - $ref: '#/components/parameters/zip_code' + responses: + '200': + description: successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/StatesResult' + /trends: + get: + tags: + - Trends + summary: "List trends" + description: "Return specific aggregations for a search" + parameters: + - $ref: '#/components/parameters/search_term' + - $ref: '#/components/parameters/field' + - $ref: '#/components/parameters/company' + - $ref: '#/components/parameters/company_public_response' + - $ref: '#/components/parameters/company_received_max' + - $ref: '#/components/parameters/company_received_min' + - $ref: '#/components/parameters/company_response' + - $ref: '#/components/parameters/date_received_max' + - $ref: '#/components/parameters/date_received_min' + - $ref: '#/components/parameters/focus' + - $ref: '#/components/parameters/has_narrative' + - $ref: '#/components/parameters/issue' + - $ref: '#/components/parameters/lens' + - $ref: '#/components/parameters/product' + - $ref: '#/components/parameters/state' + - $ref: '#/components/parameters/submitted_via' + - $ref: '#/components/parameters/sub_lens' + - $ref: '#/components/parameters/sub_lens_depth' + - $ref: '#/components/parameters/tags' + - $ref: '#/components/parameters/timely' + - $ref: '#/components/parameters/trend_depth' + - $ref: '#/components/parameters/trend_interval' + - $ref: '#/components/parameters/zip_code' + responses: + '200': + description: successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/TrendsResult' + text/csv: + schema: + $ref: '#/components/schemas/TrendsResult' + '400': + description: Invalid status value +tags: + - name: Complaints + description: These endpoints provide access to consumer complaints + - name: Trends + description: These endpoints provide access aggregated consumer complaint data + - name: Typeahead + description: These endpoints support the typeahead boxes in the UI +externalDocs: + description: Additional API Information + url: 'https://cfpb.github.io/api/ccdb/' +servers: + - url: 'https://www.consumerfinance.gov/data-research/consumer-complaints/search/api/v1/' +components: + parameters: + company: + name: company + in: query + description: Filter the results to only return these companies + explode: true + schema: + type: array + items: + type: string + company_public_response: + name: company_public_response + in: query + description: Filter the results to only return these types of public response by the company + explode: true + schema: + type: array + items: + type: string + company_received_max: + name: company_received_max + in: query + description: Return results with date < company_received_max (i.e. 2017-03-04) + schema: + type: string + format: date + company_received_min: + name: company_received_min + in: query + description: Return results with date >= company_received_min (i.e. 2017-03-04) + schema: + type: string + format: date + company_response: + name: company_response + in: query + description: Filter the results to only return these types of response by the company + explode: true + schema: + type: array + items: + type: string + date_received_max: + name: date_received_max + in: query + description: Return results with date < date_received_max (i.e. 2017-03-04) + schema: + type: string + format: date + date_received_min: + name: date_received_min + in: query + description: Return results with date >= date_received_min (i.e. 2017-03-04) + schema: + type: string + format: date + field: + name: field + in: query + description: If the parameter "search_term" has a value, use "field" to specify which field is searched. If not specified, "complaint_what_happened" will be searched. + schema: + type: string + enum: + - complaint_what_happened + - company_public_response + - all + default: complaint_what_happened + focus: + name: focus + in: query + description: The name of the product or company on which to focus charts for products and issues + explode: true + schema: + type: array + items: + type: string + format: + name: format + in: query + description: Format to be returned, if this parameter is not specified, frm/size parameters can be used properly, but if a format is specified for exporting, frm/size will be ignored + schema: + type: string + enum: + - csv + default: csv + from: + name: frm + in: query + description: Return results starting from a specific index, only if format parameter is not specified, ignore otherwise + schema: + type: integer + format: int64 + minimum: 1 + maximum: 100000 + default: 0 + has_narrative: + name: has_narrative + in: query + description: Filter the results to only return the specified state of whether it has narrative in the complaint or not, i.e. yes, no + explode: true + schema: + type: array + items: + type: string + issue: + name: issue + in: query + description: 'Filter the results to only return these types of issue and subissue, i.e. product-only: Getting a Loan, subproduct needs to include product, separated by ''•'', Getting a Loan•Can''t qualify for a loan' + explode: true + schema: + type: array + items: + type: string + lens: + name: lens + in: query + required: true + description: The data lens through which to view complaint trends over time. + schema: + type: string + enum: + - overview + - issue + - product + - tags + default: overview + no_aggs: + name: no_aggs + in: query + description: Include aggregations in result or not, True means no aggregations will be included, False means aggregations will be included. + schema: + type: boolean + default: false + no_highlight: + name: no_highlight + in: query + description: Include highlight of search term in result or not, True means no highlighting will be included, False means highlighting will be included. + schema: + type: boolean + default: false + product: + name: product + in: query + description: 'Filter the results to only return these types of product and subproduct, i.e. product-only: Mortgage, subproduct needs to include product, separated by ''•'', Mortgage•FHA mortgage' + explode: true + schema: + type: array + items: + type: string + search_after: + name: search_after + in: query + description: Used in conjunction with frm parameter to paginate results. This value is calculated by combining the values from the break_points object in the _meta key from the api response. For instance to paginate to the nth page, use the value from the break_points n:[k, id] to pass to the API the value page=n&frm=25&search_after=k_id Please see https://github.com/cfpb/cfpb.github.io/issues/292 for more detailed examples + schema: + type: string + search_term: + name: search_term + in: query + description: Return results containing specific term + schema: + type: string + size: + name: size + in: query + description: Limit the size of the results + schema: + type: integer + format: int64 + minimum: 1 + maximum: 100 + default: 10 + sort: + name: sort + in: query + description: Return results sort in a particular order + schema: + type: string + enum: + - relevance_desc + - relevance_asc + - created_date_desc + - created_date_asc + default: relevance_desc + state: + name: state + in: query + description: Filter the results to only return these states (use abbreviation, i.e. CA, VA) + explode: true + schema: + type: array + items: + type: string + sub_lens: + name: sub_lens + in: query + description: The sub-lens through which to view complaint trends over time. + schema: + type: string + enum: + - issue + - product + - sub_product + - sub_issue + - tags + sub_lens_depth: + name: sub_lens_depth + in: query + description: The top X trend sub aggregations will be returned, where X is the supplied sub_lens_depth. + schema: + type: integer + maximum: 10000000 + minimum: 5 + format: int64 + default: 10 + submitted_via: + name: submitted_via + in: query + description: Filter the results to only return these types of way consumers submitted their complaints + explode: true + schema: + type: array + items: + type: string + suggest_text: + name: text + in: query + description: text to use for suggestions + required: true + schema: + type: string + tags: + name: tags + in: query + description: Filter the results to only return these types of tag + explode: true + schema: + type: array + items: + type: string + timely: + name: timely + in: query + description: Filter the results to show whether a response was timely + explode: true + schema: + type: array + items: + type: string + trend_depth: + name: trend_depth + in: query + description: The top X trend aggregations will be returned, where X is the supplied trend_depth. + schema: + type: integer + maximum: 10000000 + minimum: 5 + format: int64 + default: 10 + trend_interval: + name: trend_interval + in: query + description: The interval of time to use for trends aggregations histograms. When using day intervals, we recommend querying for date_received_min / max periods of less than one year. + required: true + schema: + type: string + enum: + - year + - quarter + - month + - week + - day + zip_code: + name: zip_code + in: query + description: Filter the results to only return these zip codes + explode: true + schema: + type: array + items: + type: string + schemas: + Aggregation: + type: object + description: An OpenSearch aggregation + properties: + doc_count: + type: integer + description: The total number of complaints covered in this aggregation + field: + type: object + description: The name of the ` being aggregated + properties: + buckets: + type: array + items: + $ref: '#/components/schemas/Bucket' + doc_count_error_upper_bound: + type: integer + description: The number of possible errors that occurred when searching the shards + sum_other_doc_count: + type: integer + description: The number of complaints that were not included in this aggregation. + AggregationDate: + type: object + properties: + value_as_string: + type: string + format: date-time + description: ISO-8601 formatted date (yyyy-mm-ddTHH:MM:SSZZ) + value: + type: number + description: Seconds since 1970 (Unix Epoch) + Bucket: + type: object + properties: + doc_count: + type: integer + description: The number of complaints that match this key + key: + type: string + Complaint: + type: object + externalDocs: + description: Official documentation + url: 'https://cfpb.github.io/api/ccdb/fields.html' + properties: + company: + type: string + description: The complaint is about this company + company_public_response: + type: string + description: 'The company''s optional, public-facing response to a consumer''s complaint' + company_response: + type: string + description: The response from the company about this complaint + complaint_id: + type: integer + description: The unique identification number for a complaint + complaint_what_happened: + type: string + description: A description of the complaint provided by the consumer + date_received: + type: string + format: date + description: The date the CFPB received the complaint + date_sent_to_company: + type: string + description: The date the CFPB sent the complaint to the company + has_narrative: + type: boolean + description: Indicates this complaint has a narrative + issue: + type: string + description: The issue the consumer identified in the complaint + product: + type: string + description: The type of product the consumer identified in the complaint + state: + type: string + description: The state of the mailing address provided by the consumer + sub_issue: + type: string + description: The sub-issue the consumer identified in the complaint + sub_product: + type: string + description: The type of sub-product the consumer identified in the complaint + submitted_via: + type: string + description: How the complaint was submitted to the CFPB + tags: + type: string + description: Data that supports easier searching and sorting of complaints + timely: + type: string + description: Indicates whether the company gave a timely response or not + zip_code: + type: string + description: The mailing ZIP code provided by the consumer + Hit: + type: object + description: A single OpenSearch result + properties: + _source: + $ref: '#/components/schemas/Complaint' + Hits: + type: object + description: A set of complaints that matched the query + properties: + hits: + type: array + items: + $ref: '#/components/schemas/Hit' + max_score: + type: number + description: The highest score in the results + format: float + total: + type: object + properties: + value: + type: integer + description: "The count of matching hits, accurate in our code even above 10,000 hits" + relation: + type: string + description: "Indicates the accuracy of the default ES response, whether the value is accurate (eq) or a lower bound (gte)" + Meta: + type: object + properties: + break_points: + type: object + description: Contains key value pairs of page and arrays. Used to paginate OpenSearch results in list view + has_data_issue: + type: boolean + description: Indicates there has been an issue with the most recent data load + is_data_stale: + type: boolean + description: Indicates the most recent data is over 5 business days old + is_narrative_stale: + type: boolean + description: Indicates the most recent narratives are over 5 business days old + last_indexed: + type: string + description: The timestamp of the most recently indexed complaint + format: dateTime + last_updated: + type: string + description: The timestamp of the most recent complaint + format: dateTime + license: + type: string + description: The open source license under which the API operates + total_record_count: + type: integer + description: The total number of complaints currently indexed + MultiLevelAggregation: + properties: + doc_count: + type: integer + description: The total number of complaints covered in this aggregation + field: + type: object + description: The name of the field being aggregated + properties: + buckets: + type: array + items: + $ref: '#/components/schemas/MultiLevelBucket' + doc_count_error_upper_bound: + type: integer + description: The number of possible errors that occurred when searching the shards + sum_other_doc_count: + type: integer + description: The number of complaints that were not included in this aggregation. + MultiLevelBucket: + type: object + properties: + doc_count: + type: integer + description: The number of complaints that match this key + field.raw: + type: object + description: The next level of aggregations + properties: + buckets: + type: array + items: + $ref: '#/components/schemas/Aggregation' + key: + type: string + SearchResult: + type: object + properties: + _meta: + $ref: '#/components/schemas/Meta' + aggregations: + type: object + properties: + company_public_response: + $ref: '#/components/schemas/Aggregation' + company_response: + $ref: '#/components/schemas/Aggregation' + has_narrative: + $ref: '#/components/schemas/Aggregation' + issue: + $ref: '#/components/schemas/MultiLevelAggregation' + product: + $ref: '#/components/schemas/MultiLevelAggregation' + state: + $ref: '#/components/schemas/Aggregation' + submitted_via: + $ref: '#/components/schemas/Aggregation' + tags: + $ref: '#/components/schemas/Aggregation' + timely: + $ref: '#/components/schemas/Aggregation' + zip_code: + $ref: '#/components/schemas/Aggregation' + hits: + $ref: '#/components/schemas/Hits' + StatesResult: + type: object + properties: + aggregations: + type: object + properties: + issue: + $ref: '#/components/schemas/MultiLevelAggregation' + product: + $ref: '#/components/schemas/MultiLevelAggregation' + state: + $ref: '#/components/schemas/MultiLevelAggregation' + SuggestResult: + type: array + items: + type: string + TrendsResult: + type: object + properties: + aggregations: + type: object + properties: + company: + $ref: '#/components/schemas/MultiLevelAggregation' + issue: + $ref: '#/components/schemas/MultiLevelAggregation' + product: + $ref: '#/components/schemas/MultiLevelAggregation' + sub_issue: + $ref: '#/components/schemas/MultiLevelAggregation' + sub_product: + $ref: '#/components/schemas/MultiLevelAggregation' + tags: + $ref: '#/components/schemas/MultiLevelAggregation' + links: {} + callbacks: {} +security: [] From 14eaa933181b68d4724759262eab49eee2dbb38f Mon Sep 17 00:00:00 2001 From: Wyatt Pearsall Date: Tue, 18 Aug 2026 10:22:52 -0700 Subject: [PATCH 4/6] Fixup expected results --- complaint_search/es_builders.py | 1 - .../search_agg_exclude__valid.json | 20 +- .../search_no_highlight__valid.json | 18 - .../search_no_param__valid.json | 20 +- .../search_with_company__valid.json | 29 +- ...earch_with_company_agg_exclude__valid.json | 29 +- ...h_with_company_public_response__valid.json | 29 +- ...arch_with_company_received_max__valid.json | 28 +- ...arch_with_company_received_min__valid.json | 28 +- .../search_with_company_response__valid.json | 29 +- .../search_with_date_received_max__valid.json | 28 +- .../search_with_date_received_min__valid.json | 28 +- .../search_with_format_csv__valid.json | 5 +- .../search_with_format_json__valid.json | 40 - .../search_with_frm__valid.json | 211 -- .../search_with_has_narrative__valid.json | 305 --- .../search_with_issue__valid.json | 50 +- .../search_with_not_company__valid.json | 28 +- .../search_with_not_issue__valid.json | 32 +- .../search_with_not_product__valid.json | 50 +- .../search_with_product__valid.json | 50 +- .../search_with_search_term_field_all.json | 18 - .../search_with_search_term_match__valid.json | 29 +- ...earch_with_search_term_qsq_and__valid.json | 22 +- ...earch_with_search_term_qsq_not__valid.json | 22 +- ...search_with_search_term_qsq_or__valid.json | 22 +- ...search_with_search_term_qsq_to__valid.json | 22 +- .../search_with_size__valid.json | 20 +- .../search_with_size_corrected__valid.json | 19 +- .../search_with_sort__valid.json | 19 +- .../search_with_state__valid.json | 30 +- .../search_with_submitted_via__valid.json | 28 +- .../search_with_tags__valid.json | 29 +- .../search_with_timely__valid.json | 29 +- .../search_with_two_not__valid.json | 49 +- .../search_with_zip_code__valid.json | 30 +- ...arch_with_zip_code_agg_exclude__valid.json | 30 +- .../expected_results/states_agg__valid.json | 99 - .../states_date_filters__valid.json | 2 - .../trends_company_filter__valid.json | 1969 ----------------- .../trends_default_params__valid.json | 1921 ---------------- ...rends_exclude_and_date_filters__valid.json | 1135 ---------- .../trends_filter__valid.json | 1187 ---------- .../trends_issue_focus__valid.json | 693 ------ ...nds_issue_focus_company_filter__valid.json | 707 ------ .../trends_sub_lens_product__valid.json | 886 -------- .../trends_top_self_filter__valid.json | 857 ------- .../tests/test_view_suggest_company.py | 2 +- .../tests/test_views_suggest_company.py | 2 +- .../tests/test_views_suggest_zip.py | 2 +- 50 files changed, 43 insertions(+), 10895 deletions(-) delete mode 100644 complaint_search/tests/expected_results/search_with_format_json__valid.json delete mode 100644 complaint_search/tests/expected_results/search_with_frm__valid.json delete mode 100644 complaint_search/tests/expected_results/search_with_has_narrative__valid.json delete mode 100644 complaint_search/tests/expected_results/states_agg__valid.json delete mode 100644 complaint_search/tests/expected_results/trends_company_filter__valid.json delete mode 100644 complaint_search/tests/expected_results/trends_default_params__valid.json delete mode 100644 complaint_search/tests/expected_results/trends_exclude_and_date_filters__valid.json delete mode 100644 complaint_search/tests/expected_results/trends_filter__valid.json delete mode 100644 complaint_search/tests/expected_results/trends_issue_focus__valid.json delete mode 100644 complaint_search/tests/expected_results/trends_issue_focus_company_filter__valid.json delete mode 100644 complaint_search/tests/expected_results/trends_sub_lens_product__valid.json delete mode 100644 complaint_search/tests/expected_results/trends_top_self_filter__valid.json diff --git a/complaint_search/es_builders.py b/complaint_search/es_builders.py index 3b501700..cdc6da56 100644 --- a/complaint_search/es_builders.py +++ b/complaint_search/es_builders.py @@ -13,7 +13,6 @@ AGG_ZIPCODE_DEFAULT, DELIMITER, EXCLUDE_PREFIX, - EXPORT_FORMATS, PARAMS, SOURCE_FIELDS, ) diff --git a/complaint_search/tests/expected_results/search_agg_exclude__valid.json b/complaint_search/tests/expected_results/search_agg_exclude__valid.json index 391184a6..96a33576 100644 --- a/complaint_search/tests/expected_results/search_agg_exclude__valid.json +++ b/complaint_search/tests/expected_results/search_agg_exclude__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -93,22 +91,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_no_highlight__valid.json b/complaint_search/tests/expected_results/search_no_highlight__valid.json index 61e3f429..6b6d05cb 100644 --- a/complaint_search/tests/expected_results/search_no_highlight__valid.json +++ b/complaint_search/tests/expected_results/search_no_highlight__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -85,22 +83,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_no_param__valid.json b/complaint_search/tests/expected_results/search_no_param__valid.json index 391184a6..96a33576 100644 --- a/complaint_search/tests/expected_results/search_no_param__valid.json +++ b/complaint_search/tests/expected_results/search_no_param__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -93,22 +91,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_company__valid.json b/complaint_search/tests/expected_results/search_with_company__valid.json index 548fe3e9..7dc3a493 100644 --- a/complaint_search/tests/expected_results/search_with_company__valid.json +++ b/complaint_search/tests/expected_results/search_with_company__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -120,31 +118,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "terms": { - "company.raw": [ - "Bank 1", - "Second Bank" - ] - } - } - ], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_company_agg_exclude__valid.json b/complaint_search/tests/expected_results/search_with_company_agg_exclude__valid.json index cfe215ae..67afc994 100644 --- a/complaint_search/tests/expected_results/search_with_company_agg_exclude__valid.json +++ b/complaint_search/tests/expected_results/search_with_company_agg_exclude__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -120,31 +118,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "terms": { - "company.raw": [ - "Bank 1", - "Second Bank" - ] - } - } - ], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_company_public_response__valid.json b/complaint_search/tests/expected_results/search_with_company_public_response__valid.json index d7037b81..7531c19f 100644 --- a/complaint_search/tests/expected_results/search_with_company_public_response__valid.json +++ b/complaint_search/tests/expected_results/search_with_company_public_response__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -120,31 +118,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "terms": { - "company_public_response.raw": [ - "Company chooses not to provide a public response", - "Company believes it acted appropriately as authorized by contract or law" - ] - } - } - ], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_company_received_max__valid.json b/complaint_search/tests/expected_results/search_with_company_received_max__valid.json index d78fb3bf..f8cbe995 100644 --- a/complaint_search/tests/expected_results/search_with_company_received_max__valid.json +++ b/complaint_search/tests/expected_results/search_with_company_received_max__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -125,30 +123,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "range": { - "date_sent_to_company": { - "to": "2017-04-14" - } - } - } - ], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_company_received_min__valid.json b/complaint_search/tests/expected_results/search_with_company_received_min__valid.json index c70c67eb..2d81d3d6 100644 --- a/complaint_search/tests/expected_results/search_with_company_received_min__valid.json +++ b/complaint_search/tests/expected_results/search_with_company_received_min__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -125,30 +123,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "range": { - "date_sent_to_company": { - "from": "2014-04-14" - } - } - } - ], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_company_response__valid.json b/complaint_search/tests/expected_results/search_with_company_response__valid.json index 14311958..2fc7d623 100644 --- a/complaint_search/tests/expected_results/search_with_company_response__valid.json +++ b/complaint_search/tests/expected_results/search_with_company_response__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -120,31 +118,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "terms": { - "company_response": [ - "Closed", - "Closed with non-monetary relief" - ] - } - } - ], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_date_received_max__valid.json b/complaint_search/tests/expected_results/search_with_date_received_max__valid.json index 7b75cc53..8dfd5c05 100644 --- a/complaint_search/tests/expected_results/search_with_date_received_max__valid.json +++ b/complaint_search/tests/expected_results/search_with_date_received_max__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -125,30 +123,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "range": { - "date_received": { - "to": "2017-04-14" - } - } - } - ], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_date_received_min__valid.json b/complaint_search/tests/expected_results/search_with_date_received_min__valid.json index e7ed9090..7d8680c5 100644 --- a/complaint_search/tests/expected_results/search_with_date_received_min__valid.json +++ b/complaint_search/tests/expected_results/search_with_date_received_min__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -125,30 +123,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "range": { - "date_received": { - "from": "2014-04-14" - } - } - } - ], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_format_csv__valid.json b/complaint_search/tests/expected_results/search_with_format_csv__valid.json index 48512e97..e6858744 100644 --- a/complaint_search/tests/expected_results/search_with_format_csv__valid.json +++ b/complaint_search/tests/expected_results/search_with_format_csv__valid.json @@ -5,7 +5,6 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "issue", "product", "state", @@ -22,7 +21,7 @@ "query_string": { "query": "*", "fields": [ - "complaint_what_happened" + "*" ], "default_operator": "AND" } @@ -30,7 +29,7 @@ "highlight": { "require_field_match": false, "fields": { - "complaint_what_happened": {} + "*": {} }, "number_of_fragments": 1, "fragment_size": 500 diff --git a/complaint_search/tests/expected_results/search_with_format_json__valid.json b/complaint_search/tests/expected_results/search_with_format_json__valid.json deleted file mode 100644 index fd5b8eea..00000000 --- a/complaint_search/tests/expected_results/search_with_format_json__valid.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "size": 10, - "_source": [ - "company", - "company_public_response", - "company_response", - "complaint_id", - "complaint_what_happened", - "date_received", - "date_sent_to_company", - "issue", - "product", - "state", - "submitted_via", - "sub_issue", - "sub_product", - "tags", - "timely", - "zip_code" - ], - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "highlight": { - "require_field_match": false, - "fields": { - "complaint_what_happened": {} - }, - "number_of_fragments": 1, - "fragment_size": 500 - }, - "sort": [{"_score": {"order": "desc"}}], - "post_filter": {"bool": {"filter": [], "should": [], "must": []}} -} diff --git a/complaint_search/tests/expected_results/search_with_frm__valid.json b/complaint_search/tests/expected_results/search_with_frm__valid.json deleted file mode 100644 index 07045e34..00000000 --- a/complaint_search/tests/expected_results/search_with_frm__valid.json +++ /dev/null @@ -1,211 +0,0 @@ -{ - "from": 20, - "size": 10, - "_source": [ - "company", - "company_public_response", - "company_response", - "complaint_id", - "complaint_what_happened", - "date_received", - "date_sent_to_company", - "has_narrative", - "issue", - "product", - "state", - "submitted_via", - "sub_issue", - "sub_product", - "tags", - "timely", - "zip_code" - ], - "highlight": { - "require_field_match": false, - "number_of_fragments": 1, - "fragment_size": 500, - "fields": { - "complaint_what_happened": {} - } - }, - "sort": [ - { - "_score": { - "order": "desc" - } - } - ], - "post_filter": { - "bool": { - "must": [], - "must_not": [] - } - }, - "aggs": { - "company": { - "aggs": { - "company": { - "terms": { - "size": 6500, - "field": "company.raw" - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, - "company_public_response": { - "aggs": { - "company_public_response": { - "terms": { - "field": "company_public_response.raw" - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, - "company_response": { - "aggs": { - "company_response": { - "terms": { - "field": "company_response" - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative" - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, - "issue": { - "aggs": { - "issue": { - "terms": { - "field": "issue.raw" - }, - "aggs": { - "sub_issue.raw": { - "terms": { - "field": "sub_issue.raw" - } - } - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, - "product": { - "aggs": { - "product": { - "terms": { - "field": "product.raw" - }, - "aggs": { - "sub_product.raw": { - "terms": { - "field": "sub_product.raw" - } - } - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, - "state": { - "aggs": { - "state": { - "terms": { - "field": "state" - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, - "submitted_via": { - "aggs": { - "submitted_via": { - "terms": { - "field": "submitted_via" - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, - "tags": { - "aggs": { - "tags": { - "terms": { - "field": "tags" - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, - "timely": { - "aggs": { - "timely": { - "terms": { - "field": "timely" - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - } - } -} diff --git a/complaint_search/tests/expected_results/search_with_has_narrative__valid.json b/complaint_search/tests/expected_results/search_with_has_narrative__valid.json deleted file mode 100644 index 71a72f63..00000000 --- a/complaint_search/tests/expected_results/search_with_has_narrative__valid.json +++ /dev/null @@ -1,305 +0,0 @@ -{ - "size": 25, - "track_total_hits": true, - "_source": [ - "company", - "company_public_response", - "company_response", - "complaint_id", - "complaint_what_happened", - "date_received", - "date_sent_to_company", - "has_narrative", - "issue", - "product", - "state", - "submitted_via", - "sub_issue", - "sub_product", - "tags", - "timely", - "zip_code" - ], - "highlight": { - "require_field_match": false, - "number_of_fragments": 1, - "fragment_size": 500, - "fields": { - "complaint_what_happened": {} - } - }, - "sort": [ - { - "_score": { - "order": "desc" - } - }, - { - "_id": "desc" - } - ], - "post_filter": { - "bool": { - "must": [ - { - "terms": { - "has_narrative": [ - "true" - ] - } - } - ], - "must_not": [] - } - }, - "aggs": { - "company": { - "aggs": { - "company": { - "terms": { - "size": 6500, - "field": "company.raw" - } - } - }, - "filter": { - "bool": { - "must": [ - { - "terms": { - "has_narrative": [ - "true" - ] - } - } - ], - "must_not": [] - } - } - }, - "company_public_response": { - "aggs": { - "company_public_response": { - "terms": { - "field": "company_public_response.raw", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "terms": { - "has_narrative": [ - "true" - ] - } - } - ], - "must_not": [] - } - } - }, - "company_response": { - "aggs": { - "company_response": { - "terms": { - "field": "company_response", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "terms": { - "has_narrative": [ - "true" - ] - } - } - ], - "must_not": [] - } - } - }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, - "issue": { - "aggs": { - "issue": { - "terms": { - "size": 200, - "field": "issue.raw" - }, - "aggs": { - "sub_issue.raw": { - "terms": { - "field": "sub_issue.raw", - "size": 250 - } - } - } - } - }, - "filter": { - "bool": { - "must": [ - { - "terms": { - "has_narrative": [ - "true" - ] - } - } - ], - "must_not": [] - } - } - }, - "product": { - "aggs": { - "product": { - "terms": { - "field": "product.raw", - "size": 30 - }, - "aggs": { - "sub_product.raw": { - "terms": { - "field": "sub_product.raw", - "size": 90 - } - } - } - } - }, - "filter": { - "bool": { - "must": [ - { - "terms": { - "has_narrative": [ - "true" - ] - } - } - ], - "must_not": [] - } - } - }, - "state": { - "aggs": { - "state": { - "terms": { - "field": "state", - "size": 100 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "terms": { - "has_narrative": [ - "true" - ] - } - } - ], - "must_not": [] - } - } - }, - "submitted_via": { - "aggs": { - "submitted_via": { - "terms": { - "field": "submitted_via", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "terms": { - "has_narrative": [ - "true" - ] - } - } - ], - "must_not": [] - } - } - }, - "tags": { - "aggs": { - "tags": { - "terms": { - "field": "tags", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "terms": { - "has_narrative": [ - "true" - ] - } - } - ], - "must_not": [] - } - } - }, - "timely": { - "aggs": { - "timely": { - "terms": { - "field": "timely", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "terms": { - "has_narrative": [ - "true" - ] - } - } - ], - "must_not": [] - } - } - } - } -} diff --git a/complaint_search/tests/expected_results/search_with_issue__valid.json b/complaint_search/tests/expected_results/search_with_issue__valid.json index a3a48753..cc7911d7 100644 --- a/complaint_search/tests/expected_results/search_with_issue__valid.json +++ b/complaint_search/tests/expected_results/search_with_issue__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -213,52 +211,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "bool": { - "should": [ - { - "bool": { - "must": [ - { - "term": { - "issue.raw": "Communication tactics" - } - }, - { - "terms": { - "sub_issue.raw": [ - "Frequent or repeated calls" - ] - } - } - ] - } - }, - { - "term": { - "issue.raw": "Loan servicing, payments, escrow account" - } - } - ] - } - } - ], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_not_company__valid.json b/complaint_search/tests/expected_results/search_with_not_company__valid.json index 55156a8b..b5bc1958 100644 --- a/complaint_search/tests/expected_results/search_with_not_company__valid.json +++ b/complaint_search/tests/expected_results/search_with_not_company__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -125,30 +123,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [ - { - "terms": { - "company.raw": [ - "EQUIFAX, INC." - ] - } - } - ] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_not_issue__valid.json b/complaint_search/tests/expected_results/search_with_not_issue__valid.json index 107acbd7..de92401c 100644 --- a/complaint_search/tests/expected_results/search_with_not_issue__valid.json +++ b/complaint_search/tests/expected_results/search_with_not_issue__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -141,34 +139,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [ - { - "bool": { - "should": [ - { - "term": { - "issue.raw": "Incorrect information on your report" - } - } - ] - } - } - ] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_not_product__valid.json b/complaint_search/tests/expected_results/search_with_not_product__valid.json index fd0273b7..050bc206 100644 --- a/complaint_search/tests/expected_results/search_with_not_product__valid.json +++ b/complaint_search/tests/expected_results/search_with_not_product__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -213,52 +211,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [ - { - "bool": { - "should": [ - { - "term": { - "product.raw": "Credit reporting, credit repair services, or other personal consumer reports" - } - }, - { - "bool": { - "must": [ - { - "term": { - "product.raw": "Mortgage" - } - }, - { - "terms": { - "sub_product.raw": [ - "FHA mortgage" - ] - } - } - ] - } - } - ] - } - } - ] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_product__valid.json b/complaint_search/tests/expected_results/search_with_product__valid.json index 9a7f7cd7..941e8101 100644 --- a/complaint_search/tests/expected_results/search_with_product__valid.json +++ b/complaint_search/tests/expected_results/search_with_product__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -213,52 +211,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "bool": { - "should": [ - { - "term": { - "product.raw": "Payday loan" - } - }, - { - "bool": { - "must": [ - { - "term": { - "product.raw": "Mortgage" - } - }, - { - "terms": { - "sub_product.raw": [ - "FHA mortgage" - ] - } - } - ] - } - } - ] - } - } - ], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_search_term_field_all.json b/complaint_search/tests/expected_results/search_with_search_term_field_all.json index 50b49ea3..5c1e7fb4 100644 --- a/complaint_search/tests/expected_results/search_with_search_term_field_all.json +++ b/complaint_search/tests/expected_results/search_with_search_term_field_all.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -100,22 +98,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_search_term_match__valid.json b/complaint_search/tests/expected_results/search_with_search_term_match__valid.json index 44a0a593..5c1e7fb4 100644 --- a/complaint_search/tests/expected_results/search_with_search_term_match__valid.json +++ b/complaint_search/tests/expected_results/search_with_search_term_match__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -21,11 +19,10 @@ "zip_code" ], "query": { - "match": { - "complaint_what_happened": { - "query": "test term", - "operator": "and" - } + "query_string": { + "query": "test term", + "default_field": "*", + "default_operator": "AND" } }, "highlight": { @@ -33,7 +30,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -101,22 +98,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_search_term_qsq_and__valid.json b/complaint_search/tests/expected_results/search_with_search_term_qsq_and__valid.json index be4b3cef..d50f5d2e 100644 --- a/complaint_search/tests/expected_results/search_with_search_term_qsq_and__valid.json +++ b/complaint_search/tests/expected_results/search_with_search_term_qsq_and__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -23,7 +21,7 @@ "query": { "query_string": { "query": "test AND term", - "default_field": "complaint_what_happened" + "default_field": "*" } }, "highlight": { @@ -31,7 +29,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -99,22 +97,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_search_term_qsq_not__valid.json b/complaint_search/tests/expected_results/search_with_search_term_qsq_not__valid.json index 9b04bfc3..6281e690 100644 --- a/complaint_search/tests/expected_results/search_with_search_term_qsq_not__valid.json +++ b/complaint_search/tests/expected_results/search_with_search_term_qsq_not__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -23,7 +21,7 @@ "query": { "query_string": { "query": "test NOT term", - "default_field": "complaint_what_happened" + "default_field": "*" } }, "highlight": { @@ -31,7 +29,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -99,22 +97,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_search_term_qsq_or__valid.json b/complaint_search/tests/expected_results/search_with_search_term_qsq_or__valid.json index 7426dd50..ed39bee6 100644 --- a/complaint_search/tests/expected_results/search_with_search_term_qsq_or__valid.json +++ b/complaint_search/tests/expected_results/search_with_search_term_qsq_or__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -23,7 +21,7 @@ "query": { "query_string": { "query": "test OR term", - "default_field": "complaint_what_happened" + "default_field": "*" } }, "highlight": { @@ -31,7 +29,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -99,22 +97,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_search_term_qsq_to__valid.json b/complaint_search/tests/expected_results/search_with_search_term_qsq_to__valid.json index 86448d03..67a1ed93 100644 --- a/complaint_search/tests/expected_results/search_with_search_term_qsq_to__valid.json +++ b/complaint_search/tests/expected_results/search_with_search_term_qsq_to__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -23,7 +21,7 @@ "query": { "query_string": { "query": "term TO test", - "default_field": "complaint_what_happened" + "default_field": "*" } }, "highlight": { @@ -31,7 +29,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -99,22 +97,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_size__valid.json b/complaint_search/tests/expected_results/search_with_size__valid.json index 59005ae7..0ddc5115 100644 --- a/complaint_search/tests/expected_results/search_with_size__valid.json +++ b/complaint_search/tests/expected_results/search_with_size__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -93,22 +91,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_size_corrected__valid.json b/complaint_search/tests/expected_results/search_with_size_corrected__valid.json index 8cd40e65..e2da9b6b 100644 --- a/complaint_search/tests/expected_results/search_with_size_corrected__valid.json +++ b/complaint_search/tests/expected_results/search_with_size_corrected__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -85,21 +83,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative" - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_sort__valid.json b/complaint_search/tests/expected_results/search_with_sort__valid.json index 9a7a254e..f66f429b 100644 --- a/complaint_search/tests/expected_results/search_with_sort__valid.json +++ b/complaint_search/tests/expected_results/search_with_sort__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "all": {} } }, "sort": [ @@ -85,21 +83,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative" - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_state__valid.json b/complaint_search/tests/expected_results/search_with_state__valid.json index dd629f00..ca837dbb 100644 --- a/complaint_search/tests/expected_results/search_with_state__valid.json +++ b/complaint_search/tests/expected_results/search_with_state__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -133,32 +131,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "terms": { - "state": [ - "CA", - "VA", - "OR" - ] - } - } - ], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_submitted_via__valid.json b/complaint_search/tests/expected_results/search_with_submitted_via__valid.json index e08b9638..57d139bc 100644 --- a/complaint_search/tests/expected_results/search_with_submitted_via__valid.json +++ b/complaint_search/tests/expected_results/search_with_submitted_via__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -125,30 +123,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "terms": { - "submitted_via": [ - "Web" - ] - } - } - ], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_tags__valid.json b/complaint_search/tests/expected_results/search_with_tags__valid.json index fe1d512c..9d4207fb 100644 --- a/complaint_search/tests/expected_results/search_with_tags__valid.json +++ b/complaint_search/tests/expected_results/search_with_tags__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -129,31 +127,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "terms": { - "tags": [ - "Older American", - "Servicemember" - ] - } - } - ], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_timely__valid.json b/complaint_search/tests/expected_results/search_with_timely__valid.json index b3afdab4..f79f7957 100644 --- a/complaint_search/tests/expected_results/search_with_timely__valid.json +++ b/complaint_search/tests/expected_results/search_with_timely__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -129,31 +127,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "terms": { - "timely": [ - "Yes", - "No" - ] - } - } - ], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_two_not__valid.json b/complaint_search/tests/expected_results/search_with_two_not__valid.json index 2852e9e7..68b19292 100644 --- a/complaint_search/tests/expected_results/search_with_two_not__valid.json +++ b/complaint_search/tests/expected_results/search_with_two_not__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -209,51 +207,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [ - { - "bool": { - "must": [ - { - "bool": { - "should": [ - { - "term": { - "issue.raw": "Incorrect information on your report" - } - } - ] - } - }, - { - "bool": { - "should": [ - { - "term": { - "product.raw": "Credit reporting, credit repair services, or other personal consumer reports" - } - } - ] - } - } - ] - } - } - ] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_zip_code__valid.json b/complaint_search/tests/expected_results/search_with_zip_code__valid.json index 2ac50021..06f14c93 100644 --- a/complaint_search/tests/expected_results/search_with_zip_code__valid.json +++ b/complaint_search/tests/expected_results/search_with_zip_code__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -133,32 +131,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "field": "has_narrative", - "size": 10 - } - } - }, - "filter": { - "bool": { - "must": [ - { - "terms": { - "zip_code": [ - "12345", - "23435", - "03433" - ] - } - } - ], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/search_with_zip_code_agg_exclude__valid.json b/complaint_search/tests/expected_results/search_with_zip_code_agg_exclude__valid.json index 0142fd00..72acc919 100644 --- a/complaint_search/tests/expected_results/search_with_zip_code_agg_exclude__valid.json +++ b/complaint_search/tests/expected_results/search_with_zip_code_agg_exclude__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", @@ -25,7 +23,7 @@ "number_of_fragments": 1, "fragment_size": 500, "fields": { - "complaint_what_happened": {} + "*": {} } }, "sort": [ @@ -133,32 +131,6 @@ } } }, - "has_narrative": { - "aggs": { - "has_narrative": { - "terms": { - "size": 10, - "field": "has_narrative" - } - } - }, - "filter": { - "bool": { - "must": [ - { - "terms": { - "zip_code": [ - "12345", - "23435", - "03433" - ] - } - } - ], - "must_not": [] - } - } - }, "issue": { "aggs": { "issue": { diff --git a/complaint_search/tests/expected_results/states_agg__valid.json b/complaint_search/tests/expected_results/states_agg__valid.json deleted file mode 100644 index dc4c2148..00000000 --- a/complaint_search/tests/expected_results/states_agg__valid.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "size": 0, - "track_total_hits": true, - "_source": [ - "company", - "company_public_response", - "company_response", - "complaint_id", - "complaint_what_happened", - "date_received", - "date_sent_to_company", - "has_narrative", - "issue", - "product", - "state", - "submitted_via", - "sub_issue", - "sub_product", - "tags", - "timely", - "zip_code" - ], - "aggs": { - "issue": { - "filter": { - "bool": { - "must": [], - "must_not": [] - } - }, - "aggs": { - "issue": { - "terms": { - "field": "issue.raw", - "size": 5 - }, - "aggs": { - "issue": { - "terms": { - "field": "sub_issue.raw" - } - } - } - } - } - }, - "product": { - "filter": { - "bool": { - "must": [], - "must_not": [] - } - }, - "aggs": { - "product": { - "terms": { - "field": "product.raw", - "size": 5 - }, - "aggs": { - "product": { - "terms": { - "field": "sub_product.raw" - } - } - } - } - } - }, - "state": { - "filter": { - "bool": { - "must": [], - "must_not": [] - } - }, - "aggs": { - "state": { - "terms": { - "field": "state", - "size": 100 - }, - "aggs": { - "product": { - "terms": { - "field": "product.raw" - } - }, - "issue": { - "terms": { - "field": "issue.raw" - } - } - } - } - } - } - } -} diff --git a/complaint_search/tests/expected_results/states_date_filters__valid.json b/complaint_search/tests/expected_results/states_date_filters__valid.json index 9ba899b8..62371850 100644 --- a/complaint_search/tests/expected_results/states_date_filters__valid.json +++ b/complaint_search/tests/expected_results/states_date_filters__valid.json @@ -6,10 +6,8 @@ "company_public_response", "company_response", "complaint_id", - "complaint_what_happened", "date_received", "date_sent_to_company", - "has_narrative", "issue", "product", "state", diff --git a/complaint_search/tests/expected_results/trends_company_filter__valid.json b/complaint_search/tests/expected_results/trends_company_filter__valid.json deleted file mode 100644 index 5da5af33..00000000 --- a/complaint_search/tests/expected_results/trends_company_filter__valid.json +++ /dev/null @@ -1,1969 +0,0 @@ -{ - "took": 422, - "timed_out": false, - "_shards": { - "total": 5, - "successful": 5, - "failed": 0 - }, - "hits": { - "total": {"value": 1609587, "relation" : "gte"}, - "max_score": 0.0, - "hits": [] - }, - "aggregations": { - "dateRangeBrush": { - "doc_count": 167443, - "dateRangeBrush": { - "buckets": [ - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 622 - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 4776 - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 9963 - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 12003 - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 15973 - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 30726 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 30057 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 40917 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 22406 - } - ] - } - }, - "dateRangeBuckets": { - "doc_count": 1657420, - "dateRangeBuckets": { - "buckets": [ - { - "key_as_string": "2011-01-01T00:00:00.000Z", - "key": 1293840000000, - "doc_count": 2536 - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 72373 - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 108217 - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 153043 - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 168475 - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 191468 - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 242965 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 257313 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 277390 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 183640 - } - ] - } - }, - "product": { - "doc_count": 167443, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 215, - "buckets": [ - { - "key": "Credit reporting, credit repair services, or other personal consumer reports", - "doc_count": 115844, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Credit reporting", - "doc_count": 114864, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 39466, - "interval_diff": { - "value": 10668.0 - } - } - ] - } - }, - { - "key": "Other personal consumer report", - "doc_count": 855, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 204, - "interval_diff": { - "value": -22.0 - } - } - ] - } - }, - { - "key": "Credit repair services", - "doc_count": 125, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 37, - "interval_diff": { - "value": 13.0 - } - } - ] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 21745, - "interval_diff": { - "value": -17962.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 39707, - "interval_diff": { - "value": 10659.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 29048, - "interval_diff": { - "value": 3704.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 25344 - } - ] - } - }, - { - "key": "Credit reporting", - "doc_count": 48128, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 5018, - "interval_diff": { - "value": -10832.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 15850, - "interval_diff": { - "value": 3888.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 11962, - "interval_diff": { - "value": 2028.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 9934, - "interval_diff": { - "value": 5181.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 4753, - "interval_diff": { - "value": 4142.0 - } - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 611 - } - ] - } - }, - { - "key": "Debt collection", - "doc_count": 3008, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Credit card debt", - "doc_count": 969, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 404, - "interval_diff": { - "value": 96.0 - } - } - ] - } - }, - { - "key": "Other debt", - "doc_count": 796, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 261, - "interval_diff": { - "value": 74.0 - } - } - ] - } - }, - { - "key": "I do not know", - "doc_count": 593, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 198, - "interval_diff": { - "value": -25.0 - } - } - ] - } - }, - { - "key": "Medical debt", - "doc_count": 428, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 174, - "interval_diff": { - "value": 22.0 - } - } - ] - } - }, - { - "key": "Auto debt", - "doc_count": 76, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 34, - "interval_diff": { - "value": 8.0 - } - } - ] - } - }, - { - "key": "Other (i.e. phone, health club, etc.)", - "doc_count": 40, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 23, - "interval_diff": { - "value": 14.0 - } - } - ] - } - }, - { - "key": "Federal student loan debt", - "doc_count": 35, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 16, - "interval_diff": { - "value": 7.0 - } - } - ] - } - }, - { - "key": "Mortgage debt", - "doc_count": 21, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 6, - "interval_diff": { - "value": -1.0 - } - } - ] - } - }, - { - "key": "Medical", - "doc_count": 17, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 13, - "interval_diff": { - "value": 13.0 - } - } - ] - } - }, - { - "key": "Private student loan debt", - "doc_count": 13, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 5, - "interval_diff": { - "value": 1.0 - } - } - ] - } - }, - { - "key": "Payday loan debt", - "doc_count": 9, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 6, - "interval_diff": { - "value": 5.0 - } - } - ] - } - }, - { - "key": "Credit card", - "doc_count": 5, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 2, - "interval_diff": { - "value": 2.0 - } - } - ] - } - }, - { - "key": "Auto", - "doc_count": 2, - "trend_period": { - "buckets": [ - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 1 - } - ] - } - }, - { - "key": "Non-federal student loan", - "doc_count": 2, - "trend_period": { - "buckets": [] - } - }, - { - "key": "Federal student loan", - "doc_count": 1, - "trend_period": { - "buckets": [] - } - }, - { - "key": "Mortgage", - "doc_count": 1, - "trend_period": { - "buckets": [] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 633, - "interval_diff": { - "value": -471.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1104, - "interval_diff": { - "value": 187.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 917, - "interval_diff": { - "value": 625.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 292, - "interval_diff": { - "value": 249.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 43, - "interval_diff": { - "value": 31.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 12, - "interval_diff": { - "value": 6.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 6, - "interval_diff": { - "value": 5.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 1 - } - ] - } - }, - { - "key": "Credit card or prepaid card", - "doc_count": 170, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "General-purpose credit card or charge card", - "doc_count": 144, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 58, - "interval_diff": { - "value": -2.0 - } - } - ] - } - }, - { - "key": "Store credit card", - "doc_count": 24, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 5, - "interval_diff": { - "value": -1.0 - } - } - ] - } - }, - { - "key": "General-purpose prepaid card", - "doc_count": 1, - "trend_period": { - "buckets": [] - } - }, - { - "key": "Government benefit card", - "doc_count": 1, - "trend_period": { - "buckets": [] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 17, - "interval_diff": { - "value": -47.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 64, - "interval_diff": { - "value": -3.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 67, - "interval_diff": { - "value": 45.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 22 - } - ] - } - }, - { - "key": "Mortgage", - "doc_count": 78, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Other mortgage", - "doc_count": 23, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 11, - "interval_diff": { - "value": 4.0 - } - } - ] - } - }, - { - "key": "Conventional home mortgage", - "doc_count": 21, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 10, - "interval_diff": { - "value": 5.0 - } - } - ] - } - }, - { - "key": "Conventional fixed mortgage", - "doc_count": 12, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 4, - "interval_diff": { - "value": 3.0 - } - } - ] - } - }, - { - "key": "FHA mortgage", - "doc_count": 8, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1, - "interval_diff": { - "value": -1.0 - } - } - ] - } - }, - { - "key": "Other type of mortgage", - "doc_count": 7, - "trend_period": { - "buckets": [ - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 0, - "interval_diff": { - "value": -2.0 - } - } - ] - } - }, - { - "key": "Home equity loan or line of credit", - "doc_count": 3, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 0, - "interval_diff": { - "value": 0.0 - } - } - ] - } - }, - { - "key": "VA mortgage", - "doc_count": 3, - "trend_period": { - "buckets": [ - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 0, - "interval_diff": { - "value": 0.0 - } - } - ] - } - }, - { - "key": "Conventional adjustable mortgage (ARM)", - "doc_count": 1, - "trend_period": { - "buckets": [] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 4, - "interval_diff": { - "value": -13.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 17, - "interval_diff": { - "value": 10.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 7, - "interval_diff": { - "value": -5.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 12, - "interval_diff": { - "value": -4.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 16, - "interval_diff": { - "value": 6.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 10, - "interval_diff": { - "value": 6.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 4, - "interval_diff": { - "value": 1.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 3, - "interval_diff": { - "value": -2.0 - } - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 5 - } - ] - } - } - ] - } - }, - "max_date": { - "value": 1590512400000.0, - "value_as_string": "2020-05-26T12:00:00-05:00" - }, - "issue": { - "doc_count": 167443, - "issue": { - "doc_count_error_upper_bound": 23, - "sum_other_doc_count": 18544, - "buckets": [ - { - "key": "Incorrect information on your report", - "doc_count": 69251, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 15713, - "interval_diff": { - "value": -9935.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 25648, - "interval_diff": { - "value": 7672.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 17976, - "interval_diff": { - "value": 8062.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 9914 - } - ] - }, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Information belongs to someone else", - "doc_count": 43112, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 18048, - "interval_diff": { - "value": 8795.0 - } - } - ] - } - }, - { - "key": "Account status incorrect", - "doc_count": 8419, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2237, - "interval_diff": { - "value": -869.0 - } - } - ] - } - }, - { - "key": "Account information incorrect", - "doc_count": 8155, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2479, - "interval_diff": { - "value": -256.0 - } - } - ] - } - }, - { - "key": "Personal information incorrect", - "doc_count": 3535, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1231, - "interval_diff": { - "value": 203.0 - } - } - ] - } - }, - { - "key": "Public record information inaccurate", - "doc_count": 2492, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 668, - "interval_diff": { - "value": -68.0 - } - } - ] - } - }, - { - "key": "Old information reappears or never goes away", - "doc_count": 2249, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 568, - "interval_diff": { - "value": -150.0 - } - } - ] - } - }, - { - "key": "Information is missing that should be on the report", - "doc_count": 1158, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 381, - "interval_diff": { - "value": 29.0 - } - } - ] - } - }, - { - "key": "Information is incorrect", - "doc_count": 97, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 28, - "interval_diff": { - "value": -7.0 - } - } - ] - } - }, - { - "key": "Information that should be on the report is missing", - "doc_count": 28, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 7, - "interval_diff": { - "value": -4.0 - } - } - ] - } - } - ] - } - }, - { - "key": "Incorrect information on credit report", - "doc_count": 34468, - "trend_period": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 3569, - "interval_diff": { - "value": -8081.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 11650, - "interval_diff": { - "value": 2811.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 8839, - "interval_diff": { - "value": 1837.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 7002, - "interval_diff": { - "value": 3985.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 3017, - "interval_diff": { - "value": 2626.0 - } - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 391 - } - ] - }, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Account status", - "doc_count": 12037, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 3503, - "interval_diff": { - "value": 377.0 - } - } - ] - } - }, - { - "key": "Information is not mine", - "doc_count": 11017, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 4074, - "interval_diff": { - "value": 1352.0 - } - } - ] - } - }, - { - "key": "Account terms", - "doc_count": 3670, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 1229, - "interval_diff": { - "value": 282.0 - } - } - ] - } - }, - { - "key": "Public record", - "doc_count": 3454, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 1243, - "interval_diff": { - "value": 322.0 - } - } - ] - } - }, - { - "key": "Personal information", - "doc_count": 2476, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 958, - "interval_diff": { - "value": 256.0 - } - } - ] - } - }, - { - "key": "Reinserted previously deleted info", - "doc_count": 1814, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 643, - "interval_diff": { - "value": 222.0 - } - } - ] - } - } - ] - } - }, - { - "key": "Problem with a credit reporting company's investigation into an existing problem", - "doc_count": 24973, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 4180, - "interval_diff": { - "value": -4999.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 9179, - "interval_diff": { - "value": 2397.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 6782, - "interval_diff": { - "value": 1950.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 4832 - } - ] - }, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Their investigation did not fix an error on your report", - "doc_count": 18004, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 7031, - "interval_diff": { - "value": 2076.0 - } - } - ] - } - }, - { - "key": "Investigation took more than 30 days", - "doc_count": 2401, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 743, - "interval_diff": { - "value": 172.0 - } - } - ] - } - }, - { - "key": "Was not notified of investigation status or results", - "doc_count": 2037, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 625, - "interval_diff": { - "value": 162.0 - } - } - ] - } - }, - { - "key": "Difficulty submitting a dispute or getting information about a dispute over the phone", - "doc_count": 1680, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 527, - "interval_diff": { - "value": 0.0 - } - } - ] - } - }, - { - "key": "Problem with personal statement of dispute", - "doc_count": 849, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 253, - "interval_diff": { - "value": -12.0 - } - } - ] - } - } - ] - } - }, - { - "key": "Improper use of your report", - "doc_count": 14036, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 890, - "interval_diff": { - "value": -1629.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2519, - "interval_diff": { - "value": -113.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 2632, - "interval_diff": { - "value": -5363.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 7995 - } - ] - }, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Reporting company used your report improperly", - "doc_count": 7360, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 460, - "interval_diff": { - "value": 17.0 - } - } - ] - } - }, - { - "key": "Credit inquiries on your report that you don't recognize", - "doc_count": 6528, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2017, - "interval_diff": { - "value": -126.0 - } - } - ] - } - }, - { - "key": "Report provided to employer without your written authorization", - "doc_count": 74, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 18, - "interval_diff": { - "value": -5.0 - } - } - ] - } - }, - { - "key": "Received unsolicited financial product or insurance offers after opting out", - "doc_count": 69, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 20, - "interval_diff": { - "value": -3.0 - } - } - ] - } - } - ] - } - }, - { - "key": "Credit reporting company's investigation", - "doc_count": 6171, - "trend_period": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 833, - "interval_diff": { - "value": -1301.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 2134, - "interval_diff": { - "value": 751.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 1383, - "interval_diff": { - "value": 316.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 1067, - "interval_diff": { - "value": 423.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 644, - "interval_diff": { - "value": 534.0 - } - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 110 - } - ] - }, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Problem with statement of dispute", - "doc_count": 2280, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 678, - "interval_diff": { - "value": 178.0 - } - } - ] - } - }, - { - "key": "No notice of investigation status/result", - "doc_count": 2102, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 767, - "interval_diff": { - "value": 272.0 - } - } - ] - } - }, - { - "key": "Investigation took too long", - "doc_count": 1180, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 483, - "interval_diff": { - "value": 229.0 - } - } - ] - } - }, - { - "key": "Inadequate help over the phone", - "doc_count": 609, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 206, - "interval_diff": { - "value": 72.0 - } - } - ] - } - } - ] - } - } - ] - } - }, - "min_date": { - "value": 1322758800000.0, - "value_as_string": "2011-12-01T12:00:00-05:00" - }, - "dateRangeArea": { - "doc_count": 167443, - "dateRangeArea": { - "buckets": [ - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 622 - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 4776 - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 9963 - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 12003 - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 15973 - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 30726 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 30057 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 40917 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 22406 - } - ] - } - }, - "company": { - "doc_count": 167443, - "company": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "EQUIFAX, INC.", - "doc_count": 167443, - "trend_period": { - "buckets": [ - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 622 - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 4776, - "interval_diff": { - "value": 4154.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 9963, - "interval_diff": { - "value": 5187.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 12003, - "interval_diff": { - "value": 2040.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 15973, - "interval_diff": { - "value": 3970.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 30726, - "interval_diff": { - "value": 14753.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 30057, - "interval_diff": { - "value": -669.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 40917, - "interval_diff": { - "value": 10860.0 - } - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 22406, - "interval_diff": { - "value": -18511.0 - } - } - ] - } - } - ] - } - }, - "tags": { - "doc_count": 167443, - "tags": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Servicemember", - "doc_count": 11100, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 1194, - "interval_diff": { - "value": -1471.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2665, - "interval_diff": { - "value": -90.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 2755, - "interval_diff": { - "value": 176.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 2579, - "interval_diff": { - "value": 1901.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 678, - "interval_diff": { - "value": 122.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 556, - "interval_diff": { - "value": 141.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 415, - "interval_diff": { - "value": 198.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 217, - "interval_diff": { - "value": 176.0 - } - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 41 - } - ] - } - }, - { - "key": "Older American", - "doc_count": 5615, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 317, - "interval_diff": { - "value": -416.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 733, - "interval_diff": { - "value": 300.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 433, - "interval_diff": { - "value": -285.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 718, - "interval_diff": { - "value": -416.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 1134, - "interval_diff": { - "value": 111.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 1023, - "interval_diff": { - "value": 201.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 822, - "interval_diff": { - "value": 417.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 405, - "interval_diff": { - "value": 375.0 - } - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 30 - } - ] - } - }, - { - "key": "Older American, Servicemember", - "doc_count": 1026, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 83, - "interval_diff": { - "value": -175.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 258, - "interval_diff": { - "value": 101.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 157, - "interval_diff": { - "value": -63.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 220, - "interval_diff": { - "value": 98.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 122, - "interval_diff": { - "value": 43.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 79, - "interval_diff": { - "value": 13.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 66, - "interval_diff": { - "value": 27.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 39, - "interval_diff": { - "value": 37.0 - } - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 2 - } - ] - } - } - ] - } - } - }, - "_meta": { - "date_min": "2011-12-01T12:00:00-05:00", - "date_max": "2020-05-26T12:00:00-05:00" - } -} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/trends_default_params__valid.json b/complaint_search/tests/expected_results/trends_default_params__valid.json deleted file mode 100644 index c9fdbc87..00000000 --- a/complaint_search/tests/expected_results/trends_default_params__valid.json +++ /dev/null @@ -1,1921 +0,0 @@ -{ - "took": 1574, - "timed_out": false, - "_shards": { - "total": 5, - "successful": 5, - "failed": 0 - }, - "hits": { - "total": {"value": 1609587, "relation" : "gte"}, - "max_score": 0.0, - "hits": [] - }, - "aggregations": { - "dateRangeBrush": { - "doc_count": 1609587, - "dateRangeBrush": { - "buckets": [ - { - "key_as_string": "2011-01-01T00:00:00.000Z", - "key": 1293840000000, - "doc_count": 2536 - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 72373 - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 108217 - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 153043 - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 168475 - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 191468 - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 242966 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 257315 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 277391 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 135803 - } - ] - } - }, - "dateRangeBuckets": { - "doc_count": 1657420, - "dateRangeBuckets": { - "buckets": [ - { - "key_as_string": "2011-01-01T00:00:00.000Z", - "key": 1293840000000, - "doc_count": 2536 - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 72373 - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 108217 - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 153043 - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 168475 - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 191468 - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 242965 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 257313 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 277390 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 183640 - } - ] - } - }, - "product": { - "doc_count": 1609587, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 379312, - "buckets": [ - { - "key": "Credit reporting, credit repair services, or other personal consumer reports", - "doc_count": 403808, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Credit reporting", - "doc_count": 397258, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 136459, - "interval_diff": { - "value": 26900.0 - } - } - ] - } - }, - { - "key": "Other personal consumer report", - "doc_count": 5238, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1619, - "interval_diff": { - "value": -85.0 - } - } - ] - } - }, - { - "key": "Credit repair services", - "doc_count": 1311, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 516, - "interval_diff": { - "value": 155.0 - } - } - ] - } - }, - { - "key": "Conventional home mortgage", - "doc_count": 1, - "trend_period": { - "buckets": [] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 80195, - "interval_diff": { - "value": -58399.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 138594, - "interval_diff": { - "value": 26969.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 111625, - "interval_diff": { - "value": 38231.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 73394 - } - ] - } - }, - { - "key": "Mortgage", - "doc_count": 302238, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Other mortgage", - "doc_count": 86635, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 12942, - "interval_diff": { - "value": -382.0 - } - } - ] - } - }, - { - "key": "Conventional fixed mortgage", - "doc_count": 70613, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 13700, - "interval_diff": { - "value": -629.0 - } - } - ] - } - }, - { - "key": "Conventional home mortgage", - "doc_count": 44036, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 14043, - "interval_diff": { - "value": 18.0 - } - } - ] - } - }, - { - "key": "FHA mortgage", - "doc_count": 35373, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 3423, - "interval_diff": { - "value": -11.0 - } - } - ] - } - }, - { - "key": "Conventional adjustable mortgage (ARM)", - "doc_count": 25380, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 4638, - "interval_diff": { - "value": -493.0 - } - } - ] - } - }, - { - "key": "Home equity loan or line of credit", - "doc_count": 11624, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 2728, - "interval_diff": { - "value": 325.0 - } - } - ] - } - }, - { - "key": "Other type of mortgage", - "doc_count": 10642, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2084, - "interval_diff": { - "value": -1782.0 - } - } - ] - } - }, - { - "key": "VA mortgage", - "doc_count": 9117, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1353, - "interval_diff": { - "value": 172.0 - } - } - ] - } - }, - { - "key": "Home equity loan or line of credit (HELOC)", - "doc_count": 4931, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1453, - "interval_diff": { - "value": -263.0 - } - } - ] - } - }, - { - "key": "Reverse mortgage", - "doc_count": 3225, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 352, - "interval_diff": { - "value": -2.0 - } - } - ] - } - }, - { - "key": "Second mortgage", - "doc_count": 662, - "trend_period": { - "buckets": [ - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 0, - "interval_diff": { - "value": -117.0 - } - } - ] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 8820, - "interval_diff": { - "value": -13888.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 22708, - "interval_diff": { - "value": -1868.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 24576, - "interval_diff": { - "value": -6001.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 30577, - "interval_diff": { - "value": -10889.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 41466, - "interval_diff": { - "value": -879.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 42345, - "interval_diff": { - "value": -616.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 42961, - "interval_diff": { - "value": -6439.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 49400, - "interval_diff": { - "value": 11291.0 - } - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 38109, - "interval_diff": { - "value": 36833.0 - } - }, - { - "key_as_string": "2011-01-01T00:00:00.000Z", - "key": 1293840000000, - "doc_count": 1276 - } - ] - } - }, - { - "key": "Debt collection", - "doc_count": 294607, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "I do not know", - "doc_count": 61560, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 10142, - "interval_diff": { - "value": -578.0 - } - } - ] - } - }, - { - "key": "Other (i.e. phone, health club, etc.)", - "doc_count": 44543, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 13062, - "interval_diff": { - "value": 1030.0 - } - } - ] - } - }, - { - "key": "Other debt", - "doc_count": 43944, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 12646, - "interval_diff": { - "value": -2764.0 - } - } - ] - } - }, - { - "key": "Credit card debt", - "doc_count": 31785, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 10766, - "interval_diff": { - "value": -201.0 - } - } - ] - } - }, - { - "key": "Credit card", - "doc_count": 28698, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 7353, - "interval_diff": { - "value": -520.0 - } - } - ] - } - }, - { - "key": "Medical debt", - "doc_count": 23892, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 7604, - "interval_diff": { - "value": -519.0 - } - } - ] - } - }, - { - "key": "Medical", - "doc_count": 21187, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 6882, - "interval_diff": { - "value": 999.0 - } - } - ] - } - }, - { - "key": "Payday loan", - "doc_count": 7562, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 1668, - "interval_diff": { - "value": -425.0 - } - } - ] - } - }, - { - "key": "Auto debt", - "doc_count": 4840, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1562, - "interval_diff": { - "value": -62.0 - } - } - ] - } - }, - { - "key": "Mortgage", - "doc_count": 4809, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 1441, - "interval_diff": { - "value": 330.0 - } - } - ] - } - }, - { - "key": "Payday loan debt", - "doc_count": 4563, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1322, - "interval_diff": { - "value": -206.0 - } - } - ] - } - }, - { - "key": "Auto", - "doc_count": 3755, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 1047, - "interval_diff": { - "value": 145.0 - } - } - ] - } - }, - { - "key": "Mortgage debt", - "doc_count": 3347, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 913, - "interval_diff": { - "value": -209.0 - } - } - ] - } - }, - { - "key": "Non-federal student loan", - "doc_count": 2881, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 708, - "interval_diff": { - "value": -33.0 - } - } - ] - } - }, - { - "key": "Federal student loan debt", - "doc_count": 2501, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 804, - "interval_diff": { - "value": -63.0 - } - } - ] - } - }, - { - "key": "Federal student loan", - "doc_count": 2475, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 653, - "interval_diff": { - "value": 105.0 - } - } - ] - } - }, - { - "key": "Private student loan debt", - "doc_count": 2265, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 645, - "interval_diff": { - "value": -179.0 - } - } - ] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 18667, - "interval_diff": { - "value": -27737.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 46404, - "interval_diff": { - "value": -4781.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 51185, - "interval_diff": { - "value": 3232.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 47953, - "interval_diff": { - "value": 7487.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 40466, - "interval_diff": { - "value": 742.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 39724, - "interval_diff": { - "value": 585.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 39139, - "interval_diff": { - "value": 28070.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 11069 - } - ] - } - }, - { - "key": "Credit reporting", - "doc_count": 140432, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 16587, - "interval_diff": { - "value": -27494.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 44081, - "interval_diff": { - "value": 9809.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 34272, - "interval_diff": { - "value": 5033.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 29239, - "interval_diff": { - "value": 14859.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 14380, - "interval_diff": { - "value": 12507.0 - } - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 1873 - } - ] - } - }, - { - "key": "Credit card", - "doc_count": 89190, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 7133, - "interval_diff": { - "value": -13932.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 21065, - "interval_diff": { - "value": 3765.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 17300, - "interval_diff": { - "value": 3326.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 13974, - "interval_diff": { - "value": 869.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 13105, - "interval_diff": { - "value": -2248.0 - } - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 15353, - "interval_diff": { - "value": 14093.0 - } - }, - { - "key_as_string": "2011-01-01T00:00:00.000Z", - "key": 1293840000000, - "doc_count": 1260 - } - ] - } - } - ] - } - }, - "max_date": { - "value": 1590512400000.0, - "value_as_string": "2020-05-26T12:00:00-05:00" - }, - "issue": { - "doc_count": 1609587, - "issue": { - "doc_count_error_upper_bound": 14722, - "sum_other_doc_count": 970070, - "buckets": [ - { - "key": "Incorrect information on your report", - "doc_count": 256833, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 57664, - "interval_diff": { - "value": -33286.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 90950, - "interval_diff": { - "value": 20706.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 70244, - "interval_diff": { - "value": 32269.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 37975 - } - ] - }, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Information belongs to someone else", - "doc_count": 142692, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 56681, - "interval_diff": { - "value": 25205.0 - } - } - ] - } - }, - { - "key": "Account status incorrect", - "doc_count": 39929, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 11333, - "interval_diff": { - "value": -3107.0 - } - } - ] - } - }, - { - "key": "Account information incorrect", - "doc_count": 36563, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 11324, - "interval_diff": { - "value": -1148.0 - } - } - ] - } - }, - { - "key": "Personal information incorrect", - "doc_count": 11948, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 4276, - "interval_diff": { - "value": 833.0 - } - } - ] - } - }, - { - "key": "Old information reappears or never goes away", - "doc_count": 9426, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2465, - "interval_diff": { - "value": -731.0 - } - } - ] - } - }, - { - "key": "Public record information inaccurate", - "doc_count": 9382, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2678, - "interval_diff": { - "value": -276.0 - } - } - ] - } - }, - { - "key": "Information is missing that should be on the report", - "doc_count": 4590, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1535, - "interval_diff": { - "value": 79.0 - } - } - ] - } - }, - { - "key": "Information is incorrect", - "doc_count": 692, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 197, - "interval_diff": { - "value": -52.0 - } - } - ] - } - }, - { - "key": "Information that should be on the report is missing", - "doc_count": 116, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 41, - "interval_diff": { - "value": 5.0 - } - } - ] - } - }, - { - "key": "Incorrect information on your report", - "doc_count": 1, - "trend_period": { - "buckets": [] - } - } - ] - } - }, - { - "key": "Loan modification,collection,foreclosure", - "doc_count": 112309, - "trend_period": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 4026, - "interval_diff": { - "value": -12222.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 16248, - "interval_diff": { - "value": -2385.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 18633, - "interval_diff": { - "value": -2845.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 21478, - "interval_diff": { - "value": -7661.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 29139, - "interval_diff": { - "value": 6998.0 - } - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 22141, - "interval_diff": { - "value": 21497.0 - } - }, - { - "key_as_string": "2011-01-01T00:00:00.000Z", - "key": 1293840000000, - "doc_count": 644 - } - ] - }, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [] - } - }, - { - "key": "Incorrect information on credit report", - "doc_count": 102686, - "trend_period": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 12306, - "interval_diff": { - "value": -20264.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 32570, - "interval_diff": { - "value": 7110.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 25460, - "interval_diff": { - "value": 3868.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 21592, - "interval_diff": { - "value": 12093.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 9499, - "interval_diff": { - "value": 8240.0 - } - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 1259 - } - ] - }, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Account status", - "doc_count": 37057, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 9820, - "interval_diff": { - "value": 381.0 - } - } - ] - } - }, - { - "key": "Information is not mine", - "doc_count": 32384, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 11305, - "interval_diff": { - "value": 3551.0 - } - } - ] - } - }, - { - "key": "Account terms", - "doc_count": 10995, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 3393, - "interval_diff": { - "value": 607.0 - } - } - ] - } - }, - { - "key": "Public record", - "doc_count": 8876, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 2800, - "interval_diff": { - "value": 426.0 - } - } - ] - } - }, - { - "key": "Personal information", - "doc_count": 7529, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 2684, - "interval_diff": { - "value": 604.0 - } - } - ] - } - }, - { - "key": "Reinserted previously deleted info", - "doc_count": 5845, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 2568, - "interval_diff": { - "value": 1541.0 - } - } - ] - } - } - ] - } - }, - { - "key": "Problem with a credit reporting company's investigation into an existing problem", - "doc_count": 90356, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 16784, - "interval_diff": { - "value": -15093.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 31877, - "interval_diff": { - "value": 6034.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 25843, - "interval_diff": { - "value": 9991.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 15852 - } - ] - }, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Their investigation did not fix an error on your report", - "doc_count": 64390, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 24074, - "interval_diff": { - "value": 5508.0 - } - } - ] - } - }, - { - "key": "Investigation took more than 30 days", - "doc_count": 7884, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2172, - "interval_diff": { - "value": 145.0 - } - } - ] - } - }, - { - "key": "Was not notified of investigation status or results", - "doc_count": 7080, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2214, - "interval_diff": { - "value": 570.0 - } - } - ] - } - }, - { - "key": "Difficulty submitting a dispute or getting information about a dispute over the phone", - "doc_count": 6126, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1885, - "interval_diff": { - "value": -108.0 - } - } - ] - } - }, - { - "key": "Problem with personal statement of dispute", - "doc_count": 4460, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1376, - "interval_diff": { - "value": -112.0 - } - } - ] - } - } - ] - } - }, - { - "key": "Loan servicing, payments, escrow account", - "doc_count": 77333, - "trend_period": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 4959, - "interval_diff": { - "value": -12075.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 17034, - "interval_diff": { - "value": 760.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 16274, - "interval_diff": { - "value": 508.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 15766, - "interval_diff": { - "value": 2519.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 13247, - "interval_diff": { - "value": 3571.0 - } - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 9676, - "interval_diff": { - "value": 9299.0 - } - }, - { - "key_as_string": "2011-01-01T00:00:00.000Z", - "key": 1293840000000, - "doc_count": 377 - } - ] - }, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [] - } - } - ] - } - }, - "collections": { - "doc_count": 1609587, - "collections": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Servicemember", - "doc_count": 111850, - "trend_period": { - "buckets": [ - { - "key_as_string": "2011-01-01T00:00:00.000Z", - "key": 1293840000000, - "doc_count": 92 - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 2276, - "interval_diff": { - "value": 2184.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 4395, - "interval_diff": { - "value": 2119.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 8071, - "interval_diff": { - "value": 3676.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 9219, - "interval_diff": { - "value": 1148.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 10034, - "interval_diff": { - "value": 815.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 20692, - "interval_diff": { - "value": 10658.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 23910, - "interval_diff": { - "value": 3218.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 23381, - "interval_diff": { - "value": -529.0 - } - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 9780, - "interval_diff": { - "value": -13601.0 - } - } - ] - } - }, - { - "key": "Older American", - "doc_count": 88499, - "trend_period": { - "buckets": [ - { - "key_as_string": "2011-01-01T00:00:00.000Z", - "key": 1293840000000, - "doc_count": 260 - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 4568, - "interval_diff": { - "value": 4308.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 8441, - "interval_diff": { - "value": 3873.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 12686, - "interval_diff": { - "value": 4245.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 14553, - "interval_diff": { - "value": 1867.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 15991, - "interval_diff": { - "value": 1438.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 9748, - "interval_diff": { - "value": -6243.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 6820, - "interval_diff": { - "value": -2928.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 10881, - "interval_diff": { - "value": 4061.0 - } - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 4551, - "interval_diff": { - "value": -6330.0 - } - } - ] - } - }, - { - "key": "Older American, Servicemember", - "doc_count": 18244, - "trend_period": { - "buckets": [ - { - "key_as_string": "2011-01-01T00:00:00.000Z", - "key": 1293840000000, - "doc_count": 29 - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 539, - "interval_diff": { - "value": 510.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 1218, - "interval_diff": { - "value": 679.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 1931, - "interval_diff": { - "value": 713.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 2250, - "interval_diff": { - "value": 319.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 2390, - "interval_diff": { - "value": 140.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 2516, - "interval_diff": { - "value": 126.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 2500, - "interval_diff": { - "value": -16.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 3514, - "interval_diff": { - "value": 1014.0 - } - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 1357, - "interval_diff": { - "value": -2157.0 - } - } - ] - } - } - ] - } - }, - "min_date": { - "value": 1322758800000.0, - "value_as_string": "2011-12-01T12:00:00-05:00" - }, - "dateRangeArea": { - "doc_count": 1609587, - "dateRangeArea": { - "buckets": [ - { - "key_as_string": "2011-01-01T00:00:00.000Z", - "key": 1293840000000, - "doc_count": 2536 - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 72373 - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 108217 - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 153043 - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 168475 - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 191468 - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 242966 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 257315 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 277391 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 135803 - } - ] - } - } - }, - "_meta": { - "date_min": "2011-12-01T12:00:00-05:00", - "date_max": "2020-05-26T12:00:00-05:00" - } -} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/trends_exclude_and_date_filters__valid.json b/complaint_search/tests/expected_results/trends_exclude_and_date_filters__valid.json deleted file mode 100644 index c828bff2..00000000 --- a/complaint_search/tests/expected_results/trends_exclude_and_date_filters__valid.json +++ /dev/null @@ -1,1135 +0,0 @@ -{ - "took": 723, - "timed_out": false, - "_shards": { - "total": 5, - "successful": 5, - "failed": 0 - }, - "hits": { - "total": {"value": 1609587, "relation" : "gte"}, - "max_score": 0.0, - "hits": [] - }, - "aggregations": { - "dateRangeBrush": { - "doc_count": 278055, - "dateRangeBrush": { - "buckets": [ - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 1037 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 276736 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 282 - } - ] - } - }, - "dateRangeBuckets": { - "doc_count": 1657420, - "dateRangeBuckets": { - "buckets": [ - { - "key_as_string": "2011-01-01T00:00:00.000Z", - "key": 1293840000000, - "doc_count": 2536 - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 72373 - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 108217 - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 153043 - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 168475 - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 191468 - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 242965 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 257313 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 277390 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 183640 - } - ] - } - }, - "product": { - "doc_count": 277018, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 22017, - "buckets": [ - { - "key": "Credit reporting, credit repair services, or other personal consumer reports", - "doc_count": 138657, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Credit reporting", - "doc_count": 136531, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 136350 - } - ] - } - }, - { - "key": "Other personal consumer report", - "doc_count": 1612, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1610 - } - ] - } - }, - { - "key": "Credit repair services", - "doc_count": 514, - "trend_period": { - "buckets": [] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 183, - "interval_diff": { - "value": -138291.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 138474 - } - ] - } - }, - { - "key": "Debt collection", - "doc_count": 46250, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Other debt", - "doc_count": 12596, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 12591 - } - ] - } - }, - { - "key": "Credit card debt", - "doc_count": 10740, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 10736 - } - ] - } - }, - { - "key": "I do not know", - "doc_count": 10116, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 10107 - } - ] - } - }, - { - "key": "Medical debt", - "doc_count": 7585, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 7580 - } - ] - } - }, - { - "key": "Auto debt", - "doc_count": 1549, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1548 - } - ] - } - }, - { - "key": "Payday loan debt", - "doc_count": 1318, - "trend_period": { - "buckets": [] - } - }, - { - "key": "Mortgage debt", - "doc_count": 908, - "trend_period": { - "buckets": [] - } - }, - { - "key": "Federal student loan debt", - "doc_count": 800, - "trend_period": { - "buckets": [] - } - }, - { - "key": "Private student loan debt", - "doc_count": 638, - "trend_period": { - "buckets": [] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 24, - "interval_diff": { - "value": -46202.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 46226 - } - ] - } - }, - { - "key": "Credit card or prepaid card", - "doc_count": 25777, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "General-purpose credit card or charge card", - "doc_count": 19704, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 19682 - } - ] - } - }, - { - "key": "Store credit card", - "doc_count": 4215, - "trend_period": { - "buckets": [] - } - }, - { - "key": "General-purpose prepaid card", - "doc_count": 1014, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1012 - } - ] - } - }, - { - "key": "Government benefit card", - "doc_count": 623, - "trend_period": { - "buckets": [] - } - }, - { - "key": "Payroll card", - "doc_count": 117, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 116 - } - ] - } - }, - { - "key": "Gift card", - "doc_count": 103, - "trend_period": { - "buckets": [] - } - }, - { - "key": "Student prepaid card", - "doc_count": 1, - "trend_period": { - "buckets": [] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 25, - "interval_diff": { - "value": -25727.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 25752 - } - ] - } - }, - { - "key": "Mortgage", - "doc_count": 22626, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Conventional home mortgage", - "doc_count": 13998, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 13987 - } - ] - } - }, - { - "key": "FHA mortgage", - "doc_count": 3416, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 3411 - } - ] - } - }, - { - "key": "Other type of mortgage", - "doc_count": 2069, - "trend_period": { - "buckets": [] - } - }, - { - "key": "Home equity loan or line of credit (HELOC)", - "doc_count": 1446, - "trend_period": { - "buckets": [] - } - }, - { - "key": "VA mortgage", - "doc_count": 1348, - "trend_period": { - "buckets": [] - } - }, - { - "key": "Reverse mortgage", - "doc_count": 349, - "trend_period": { - "buckets": [] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 16, - "interval_diff": { - "value": -22594.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 22610 - } - ] - } - }, - { - "key": "Checking or savings account", - "doc_count": 21691, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Checking account", - "doc_count": 16585, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 16576 - } - ] - } - }, - { - "key": "Other banking product or service", - "doc_count": 2867, - "trend_period": { - "buckets": [] - } - }, - { - "key": "Savings account", - "doc_count": 1583, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1581 - } - ] - } - }, - { - "key": "CD (Certificate of Deposit)", - "doc_count": 643, - "trend_period": { - "buckets": [] - } - }, - { - "key": "Personal line of credit", - "doc_count": 13, - "trend_period": { - "buckets": [] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 11, - "interval_diff": { - "value": -21669.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 21680 - } - ] - } - } - ] - } - }, - "max_date": { - "value": 1590512400000.0, - "value_as_string": "2020-05-26T12:00:00-05:00" - }, - "issue": { - "doc_count": 277018, - "issue": { - "doc_count_error_upper_bound": 1865, - "sum_other_doc_count": 105576, - "buckets": [ - { - "key": "Incorrect information on your report", - "doc_count": 91015, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 126, - "interval_diff": { - "value": -90763.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 90889 - } - ] - }, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Information belongs to someone else", - "doc_count": 56743, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 56663 - } - ] - } - }, - { - "key": "Account status incorrect", - "doc_count": 11343, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 11322 - } - ] - } - }, - { - "key": "Account information incorrect", - "doc_count": 11314, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 11301 - } - ] - } - }, - { - "key": "Personal information incorrect", - "doc_count": 4277, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 4273 - } - ] - } - }, - { - "key": "Public record information inaccurate", - "doc_count": 2680, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2676 - } - ] - } - }, - { - "key": "Old information reappears or never goes away", - "doc_count": 2468, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2465 - } - ] - } - }, - { - "key": "Information is missing that should be on the report", - "doc_count": 1534, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1533 - } - ] - } - }, - { - "key": "Information is incorrect", - "doc_count": 195, - "trend_period": { - "buckets": [] - } - }, - { - "key": "Information that should be on the report is missing", - "doc_count": 41, - "trend_period": { - "buckets": [] - } - } - ] - } - }, - { - "key": "Problem with a credit reporting company's investigation into an existing problem", - "doc_count": 31846, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 45, - "interval_diff": { - "value": -31756.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 31801 - } - ] - }, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Their investigation did not fix an error on your report", - "doc_count": 24031, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 24001 - } - ] - } - }, - { - "key": "Was not notified of investigation status or results", - "doc_count": 2216, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2213 - } - ] - } - }, - { - "key": "Investigation took more than 30 days", - "doc_count": 2177, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2170 - } - ] - } - }, - { - "key": "Difficulty submitting a dispute or getting information about a dispute over the phone", - "doc_count": 1887, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1885 - } - ] - } - }, - { - "key": "Problem with personal statement of dispute", - "doc_count": 1378, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1376 - } - ] - } - } - ] - } - }, - { - "key": "Attempts to collect debt not owed", - "doc_count": 22703, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 8, - "interval_diff": { - "value": -22687.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 22695 - } - ] - }, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Debt is not yours", - "doc_count": 10950, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 10944 - } - ] - } - }, - { - "key": "Debt was result of identity theft", - "doc_count": 6472, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 6471 - } - ] - } - }, - { - "key": "Debt was paid", - "doc_count": 4406, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 4405 - } - ] - } - }, - { - "key": "Debt was already discharged in bankruptcy and is no longer owed", - "doc_count": 875, - "trend_period": { - "buckets": [] - } - } - ] - } - }, - { - "key": "Managing an account", - "doc_count": 13850, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 7, - "interval_diff": { - "value": -13836.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 13843 - } - ] - }, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Deposits and withdrawals", - "doc_count": 4842, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 4838 - } - ] - } - }, - { - "key": "Problem using a debit or ATM card", - "doc_count": 2403, - "trend_period": { - "buckets": [] - } - }, - { - "key": "Banking errors", - "doc_count": 1747, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1746 - } - ] - } - }, - { - "key": "Funds not handled or disbursed as instructed", - "doc_count": 1377, - "trend_period": { - "buckets": [] - } - }, - { - "key": "Fee problem", - "doc_count": 1196, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1195 - } - ] - } - }, - { - "key": "Problem accessing account", - "doc_count": 872, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 871 - } - ] - } - }, - { - "key": "Problem making or receiving payments", - "doc_count": 679, - "trend_period": { - "buckets": [] - } - }, - { - "key": "Cashing a check", - "doc_count": 529, - "trend_period": { - "buckets": [] - } - }, - { - "key": "Deposits or withdrawals", - "doc_count": 138, - "trend_period": { - "buckets": [] - } - }, - { - "key": "Problem with renewal", - "doc_count": 34, - "trend_period": { - "buckets": [] - } - }, - { - "key": "Problem with fees or penalties", - "doc_count": 33, - "trend_period": { - "buckets": [] - } - } - ] - } - }, - { - "key": "Improper use of your report", - "doc_count": 12028, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 11, - "interval_diff": { - "value": -12006.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 12017 - } - ] - }, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Credit inquiries on your report that you don't recognize", - "doc_count": 9360, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 9351 - } - ] - } - }, - { - "key": "Reporting company used your report improperly", - "doc_count": 2377, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2375 - } - ] - } - }, - { - "key": "Received unsolicited financial product or insurance offers after opting out", - "doc_count": 139, - "trend_period": { - "buckets": [] - } - }, - { - "key": "Report provided to employer without your written authorization", - "doc_count": 113, - "trend_period": { - "buckets": [] - } - } - ] - } - } - ] - } - }, - "collections": { - "doc_count": 277018, - "collections": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Servicemember", - "doc_count": 23353, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 23331 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 22, - "interval_diff": { - "value": -23309.0 - } - } - ] - } - }, - { - "key": "Older American", - "doc_count": 10836, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 10828 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 8, - "interval_diff": { - "value": -10820.0 - } - } - ] - } - }, - { - "key": "Older American, Servicemember", - "doc_count": 3500, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 3499 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 1, - "interval_diff": { - "value": -3498.0 - } - } - ] - } - } - ] - } - }, - "min_date": { - "value": 1322758800000.0, - "value_as_string": "2011-12-01T12:00:00-05:00" - }, - "dateRangeArea": { - "doc_count": 277018, - "dateRangeArea": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 276736 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 282 - } - ] - } - } - }, - "_meta": { - "date_min": "2011-12-01T12:00:00-05:00", - "date_max": "2020-05-26T12:00:00-05:00" - } -} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/trends_filter__valid.json b/complaint_search/tests/expected_results/trends_filter__valid.json deleted file mode 100644 index 70dd79da..00000000 --- a/complaint_search/tests/expected_results/trends_filter__valid.json +++ /dev/null @@ -1,1187 +0,0 @@ -{ - "took": 377, - "timed_out": false, - "_shards": { - "total": 5, - "successful": 5, - "failed": 0 - }, - "hits": { - "total": {"value": 1609587, "relation" : "gte"}, - "max_score": 0.0, - "hits": [] - }, - "aggregations": { - "dateRangeBrush": { - "doc_count": 278189, - "dateRangeBrush": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 37975 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 70244 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 90950 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 79020 - } - ] - } - }, - "product": { - "doc_count": 278189, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 935, - "buckets": [ - { - "key": "Credit reporting, credit repair services, or other personal consumer reports", - "doc_count": 269825, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Credit reporting", - "doc_count": 266806, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 87654, - "interval_diff": { - "value": 21238.0 - } - } - ] - } - }, - { - "key": "Other personal consumer report", - "doc_count": 3019, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 938, - "interval_diff": { - "value": -47.0 - } - } - ] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 77519, - "interval_diff": { - "value": -11073.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 88592, - "interval_diff": { - "value": 21191.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 67401, - "interval_diff": { - "value": 31088.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 36313 - } - ] - } - }, - { - "key": "Credit card or prepaid card", - "doc_count": 2839, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "General-purpose credit card or charge card", - "doc_count": 2134, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 610, - "interval_diff": { - "value": -58.0 - } - } - ] - } - }, - { - "key": "Store credit card", - "doc_count": 705, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 216, - "interval_diff": { - "value": -33.0 - } - } - ] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 569, - "interval_diff": { - "value": -257.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 826, - "interval_diff": { - "value": -91.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 917, - "interval_diff": { - "value": 390.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 527 - } - ] - } - }, - { - "key": "Vehicle loan or lease", - "doc_count": 1686, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Loan", - "doc_count": 1359, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 380, - "interval_diff": { - "value": -112.0 - } - } - ] - } - }, - { - "key": "Lease", - "doc_count": 318, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 106, - "interval_diff": { - "value": 3.0 - } - } - ] - } - }, - { - "key": "Title loan", - "doc_count": 9, - "trend_period": { - "buckets": [ - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 2 - } - ] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 302, - "interval_diff": { - "value": -191.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 493, - "interval_diff": { - "value": -104.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 597, - "interval_diff": { - "value": 303.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 294 - } - ] - } - }, - { - "key": "Mortgage", - "doc_count": 1571, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Conventional home mortgage", - "doc_count": 854, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 226, - "interval_diff": { - "value": -66.0 - } - } - ] - } - }, - { - "key": "FHA mortgage", - "doc_count": 299, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 82, - "interval_diff": { - "value": -14.0 - } - } - ] - } - }, - { - "key": "Other type of mortgage", - "doc_count": 194, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 46, - "interval_diff": { - "value": -16.0 - } - } - ] - } - }, - { - "key": "Home equity loan or line of credit (HELOC)", - "doc_count": 136, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 36, - "interval_diff": { - "value": -13.0 - } - } - ] - } - }, - { - "key": "VA mortgage", - "doc_count": 86, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 27, - "interval_diff": { - "value": 4.0 - } - } - ] - } - }, - { - "key": "Reverse mortgage", - "doc_count": 2, - "trend_period": { - "buckets": [] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 285, - "interval_diff": { - "value": -134.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 419, - "interval_diff": { - "value": -103.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 522, - "interval_diff": { - "value": 177.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 345 - } - ] - } - }, - { - "key": "Student loan", - "doc_count": 1333, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Federal student loan servicing", - "doc_count": 981, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 250, - "interval_diff": { - "value": -108.0 - } - } - ] - } - }, - { - "key": "Private student loan", - "doc_count": 352, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 86, - "interval_diff": { - "value": -47.0 - } - } - ] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 180, - "interval_diff": { - "value": -156.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 336, - "interval_diff": { - "value": -155.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 491, - "interval_diff": { - "value": 165.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 326 - } - ] - } - } - ] - } - }, - "max_date": { - "value": 1593622800000.0, - "value_as_string": "2020-07-01T12:00:00-05:00" - }, - "issue": { - "doc_count": 278189, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Incorrect information on your report", - "doc_count": 278189, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 79020, - "interval_diff": { - "value": -11930.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 90950, - "interval_diff": { - "value": 20706.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 70244, - "interval_diff": { - "value": 32269.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 37975 - } - ] - }, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Information belongs to someone else", - "doc_count": 158319, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 56681, - "interval_diff": { - "value": 25205.0 - } - } - ] - } - }, - { - "key": "Account status incorrect", - "doc_count": 41781, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 11333, - "interval_diff": { - "value": -3107.0 - } - } - ] - } - }, - { - "key": "Account information incorrect", - "doc_count": 38510, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 11324, - "interval_diff": { - "value": -1148.0 - } - } - ] - } - }, - { - "key": "Personal information incorrect", - "doc_count": 12750, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 4276, - "interval_diff": { - "value": 833.0 - } - } - ] - } - }, - { - "key": "Old information reappears or never goes away", - "doc_count": 9814, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2465, - "interval_diff": { - "value": -731.0 - } - } - ] - } - }, - { - "key": "Public record information inaccurate", - "doc_count": 9665, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2678, - "interval_diff": { - "value": -276.0 - } - } - ] - } - }, - { - "key": "Information is missing that should be on the report", - "doc_count": 4929, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1535, - "interval_diff": { - "value": 79.0 - } - } - ] - } - }, - { - "key": "Information is incorrect", - "doc_count": 727, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 197, - "interval_diff": { - "value": -52.0 - } - } - ] - } - }, - { - "key": "Information that should be on the report is missing", - "doc_count": 120, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 41, - "interval_diff": { - "value": 5.0 - } - } - ] - } - }, - { - "key": "Incorrect information on your report", - "doc_count": 1, - "trend_period": { - "buckets": [] - } - } - ] - } - } - ] - } - }, - "min_date": { - "value": 1322758800000.0, - "value_as_string": "2011-12-01T12:00:00-05:00" - }, - "dateRangeArea": { - "doc_count": 278189, - "dateRangeArea": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 37975 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 70244 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 90950 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 79020 - } - ] - } - }, - "tags": { - "doc_count": 278189, - "tags": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Servicemember", - "doc_count": 19873, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 3750, - "interval_diff": { - "value": -2224.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 5974, - "interval_diff": { - "value": -255.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 6229, - "interval_diff": { - "value": 2309.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 3920 - } - ] - } - }, - { - "key": "Older American", - "doc_count": 4025, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 846, - "interval_diff": { - "value": -720.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1566, - "interval_diff": { - "value": 646.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 920, - "interval_diff": { - "value": 227.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 693 - } - ] - } - }, - { - "key": "Older American, Servicemember", - "doc_count": 1313, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 231, - "interval_diff": { - "value": -249.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 480, - "interval_diff": { - "value": 138.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 342, - "interval_diff": { - "value": 82.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 260 - } - ] - } - } - ] - } - }, - "dateRangeBuckets": { - "doc_count": 1657420, - "dateRangeBuckets": { - "buckets": [ - { - "key_as_string": "2011-01-01T00:00:00.000Z", - "key": 1293840000000, - "doc_count": 2536 - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 72373 - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 108217 - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 153043 - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 168475 - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 191468 - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 242965 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 257313 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 277390 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 183640 - } - ] - } - } - }, - "_meta": { - "date_min": "2011-12-01T12:00:00-05:00", - "date_max": "2020-07-01T12:00:00-05:00" - }, - "body": { - "from": 0, - "size": 0, - "_source": [ - "company", - "company_public_response", - "company_response", - "complaint_id", - "complaint_what_happened", - "date_received", - "date_sent_to_company", - "has_narrative", - "issue", - "product", - "state", - "submitted_via", - "sub_issue", - "sub_product", - "tags", - "timely", - "zip_code" - ], - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "aggs": { - "dateRangeArea": { - "aggs": { - "dateRangeArea": { - "date_histogram": { - "field": "date_received", - "interval": "year" - } - } - }, - "filter": { - "bool": { - "must": [ - { - "bool": { - "should": [ - { - "term": { - "issue.raw": "Incorrect information on your report" - } - } - ] - } - } - ], - "must_not": [] - } - } - }, - "dateRangeBrush": { - "aggs": { - "dateRangeBrush": { - "date_histogram": { - "field": "date_received", - "interval": "year" - } - } - }, - "filter": { - "bool": { - "must": [ - { - "bool": { - "should": [ - { - "term": { - "issue.raw": "Incorrect information on your report" - } - } - ] - } - } - ], - "must_not": [] - } - } - }, - "issue": { - "aggs": { - "issue": { - "terms": { - "field": "issue.raw", - "size": 5 - }, - "aggs": { - "trend_period": { - "date_histogram": { - "field": "date_received", - "interval": "year" - }, - "aggs": { - "interval_diff": { - "serial_diff": { - "buckets_path": "_count" - } - } - } - }, - "issue": { - "terms": { - "field": "sub_issue.raw", - "size": 0 - }, - "aggs": { - "trend_period": { - "date_histogram": { - "field": "date_received", - "interval": "year" - }, - "aggs": { - "interval_diff": { - "serial_diff": { - "buckets_path": "_count" - } - } - } - } - } - } - } - } - }, - "filter": { - "bool": { - "must": [ - { - "bool": { - "should": [ - { - "term": { - "issue.raw": "Incorrect information on your report" - } - } - ] - } - } - ], - "must_not": [] - } - } - }, - "tags": { - "aggs": { - "tags": { - "terms": { - "field": "tags", - "size": 5 - }, - "aggs": { - "trend_period": { - "date_histogram": { - "field": "date_received", - "interval": "year" - }, - "aggs": { - "interval_diff": { - "serial_diff": { - "buckets_path": "_count" - } - } - } - } - } - } - }, - "filter": { - "bool": { - "must": [ - { - "bool": { - "should": [ - { - "term": { - "issue.raw": "Incorrect information on your report" - } - } - ] - } - } - ], - "must_not": [] - } - } - }, - "product": { - "aggs": { - "product": { - "terms": { - "field": "product.raw", - "size": 5 - }, - "aggs": { - "trend_period": { - "date_histogram": { - "field": "date_received", - "interval": "year" - }, - "aggs": { - "interval_diff": { - "serial_diff": { - "buckets_path": "_count" - } - } - } - }, - "product": { - "terms": { - "field": "sub_product.raw", - "size": 0 - }, - "aggs": { - "trend_period": { - "date_histogram": { - "field": "date_received", - "interval": "year" - }, - "aggs": { - "interval_diff": { - "serial_diff": { - "buckets_path": "_count" - } - } - } - } - } - } - } - } - }, - "filter": { - "bool": { - "must": [ - { - "bool": { - "should": [ - { - "term": { - "issue.raw": "Incorrect information on your report" - } - } - ] - } - } - ], - "must_not": [] - } - } - }, - "min_date": { - "min": { - "field": "date_received", - "format": "yyyy-MM-dd'T'12:00:00-05:00" - } - }, - "max_date": { - "max": { - "field": "date_received", - "format": "yyyy-MM-dd'T'12:00:00-05:00" - } - } - } - } -} diff --git a/complaint_search/tests/expected_results/trends_issue_focus__valid.json b/complaint_search/tests/expected_results/trends_issue_focus__valid.json deleted file mode 100644 index 891b6b1a..00000000 --- a/complaint_search/tests/expected_results/trends_issue_focus__valid.json +++ /dev/null @@ -1,693 +0,0 @@ -{ - "took": 362, - "timed_out": false, - "_shards": { - "total": 5, - "successful": 5, - "failed": 0 - }, - "hits": { - "total": {"value": 1609587, "relation" : "gte"}, - "max_score": 0.0, - "hits": [] - }, - "aggregations": { - "dateRangeBrush": { - "doc_count": 256833, - "dateRangeBrush": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 37975 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 70244 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 90950 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 57664 - } - ] - } - }, - "dateRangeBuckets": { - "doc_count": 1657420, - "dateRangeBuckets": { - "buckets": [ - { - "key_as_string": "2011-01-01T00:00:00.000Z", - "key": 1293840000000, - "doc_count": 2536 - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 72373 - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 108217 - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 153043 - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 168475 - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 191468 - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 242965 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 257313 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 277390 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 183640 - } - ] - } - }, - "product": { - "doc_count": 256833, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 881, - "buckets": [ - { - "key": "Credit reporting, credit repair services, or other personal consumer reports", - "doc_count": 248864, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 56558, - "interval_diff": { - "value": -32034.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 88592, - "interval_diff": { - "value": 21191.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 67401, - "interval_diff": { - "value": 31088.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 36313 - } - ] - } - }, - { - "key": "Credit card or prepaid card", - "doc_count": 2689, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 419, - "interval_diff": { - "value": -407.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 826, - "interval_diff": { - "value": -91.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 917, - "interval_diff": { - "value": 390.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 527 - } - ] - } - }, - { - "key": "Vehicle loan or lease", - "doc_count": 1620, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 236, - "interval_diff": { - "value": -257.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 493, - "interval_diff": { - "value": -104.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 597, - "interval_diff": { - "value": 303.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 294 - } - ] - } - }, - { - "key": "Mortgage", - "doc_count": 1492, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 206, - "interval_diff": { - "value": -213.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 419, - "interval_diff": { - "value": -103.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 522, - "interval_diff": { - "value": 177.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 345 - } - ] - } - }, - { - "key": "Student loan", - "doc_count": 1287, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 134, - "interval_diff": { - "value": -202.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 336, - "interval_diff": { - "value": -155.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 491, - "interval_diff": { - "value": 165.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 326 - } - ] - } - } - ] - } - }, - "max_date": { - "value": 1590512400000.0, - "value_as_string": "2020-05-26T12:00:00-05:00" - }, - "issue": { - "doc_count": 256833, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Incorrect information on your report", - "doc_count": 256833, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 57664, - "interval_diff": { - "value": -33286.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 90950, - "interval_diff": { - "value": 20706.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 70244, - "interval_diff": { - "value": 32269.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 37975 - } - ] - } - } - ] - } - }, - "min_date": { - "value": 1322758800000.0, - "value_as_string": "2011-12-01T12:00:00-05:00" - }, - "dateRangeArea": { - "doc_count": 256833, - "dateRangeArea": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 37975 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 70244 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 90950 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 57664 - } - ] - } - }, - "sub-issue": { - "doc_count": 256833, - "sub-issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 14781, - "buckets": [ - { - "key": "Information belongs to someone else", - "doc_count": 142692, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 41930, - "interval_diff": { - "value": -14751.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 56681, - "interval_diff": { - "value": 25205.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 31476, - "interval_diff": { - "value": 18871.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 12605 - } - ] - } - }, - { - "key": "Account status incorrect", - "doc_count": 39929, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 5267, - "interval_diff": { - "value": -6066.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 11333, - "interval_diff": { - "value": -3107.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 14440, - "interval_diff": { - "value": 5551.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 8889 - } - ] - } - }, - { - "key": "Account information incorrect", - "doc_count": 36563, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 5255, - "interval_diff": { - "value": -6069.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 11324, - "interval_diff": { - "value": -1148.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 12472, - "interval_diff": { - "value": 4960.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 7512 - } - ] - } - }, - { - "key": "Personal information incorrect", - "doc_count": 11948, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 2129, - "interval_diff": { - "value": -2147.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 4276, - "interval_diff": { - "value": 833.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 3443, - "interval_diff": { - "value": 1343.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 2100 - } - ] - } - }, - { - "key": "Old information reappears or never goes away", - "doc_count": 9426, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 1123, - "interval_diff": { - "value": -1342.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2465, - "interval_diff": { - "value": -731.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 3196, - "interval_diff": { - "value": 554.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 2642 - } - ] - } - } - ] - } - }, - "tags": { - "doc_count": 256833, - "tags": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Servicemember", - "doc_count": 18870, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 2747, - "interval_diff": { - "value": -3227.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 5974, - "interval_diff": { - "value": -255.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 6229, - "interval_diff": { - "value": 2309.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 3920 - } - ] - } - }, - { - "key": "Older American", - "doc_count": 3819, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 640, - "interval_diff": { - "value": -926.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1566, - "interval_diff": { - "value": 646.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 920, - "interval_diff": { - "value": 227.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 693 - } - ] - } - }, - { - "key": "Older American, Servicemember", - "doc_count": 1265, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 183, - "interval_diff": { - "value": -297.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 480, - "interval_diff": { - "value": 138.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 342, - "interval_diff": { - "value": 82.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 260 - } - ] - } - } - ] - } - } - }, - "_meta": { - "date_min": "2011-12-01T12:00:00-05:00", - "date_max": "2020-05-26T12:00:00-05:00" - } -} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/trends_issue_focus_company_filter__valid.json b/complaint_search/tests/expected_results/trends_issue_focus_company_filter__valid.json deleted file mode 100644 index 3f53d6e7..00000000 --- a/complaint_search/tests/expected_results/trends_issue_focus_company_filter__valid.json +++ /dev/null @@ -1,707 +0,0 @@ -{ - "took": 321, - "timed_out": false, - "_shards": { - "total": 5, - "successful": 5, - "failed": 0 - }, - "hits": { - "total": {"value": 1609587, "relation" : "gte"}, - "max_score": 0.0, - "hits": [] - }, - "aggregations": { - "dateRangeBrush": { - "doc_count": 69251, - "dateRangeBrush": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 9914 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 17976 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 25648 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 15713 - } - ] - } - }, - "dateRangeBuckets": { - "doc_count": 1657420, - "dateRangeBuckets": { - "buckets": [ - { - "key_as_string": "2011-01-01T00:00:00.000Z", - "key": 1293840000000, - "doc_count": 2536 - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 72373 - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 108217 - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 153043 - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 168475 - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 191468 - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 242965 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 257313 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 277390 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 183640 - } - ] - } - }, - "product": { - "doc_count": 69251, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 4, - "buckets": [ - { - "key": "Credit reporting, credit repair services, or other personal consumer reports", - "doc_count": 69217, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 15712, - "interval_diff": { - "value": -9930.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 25642, - "interval_diff": { - "value": 7676.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 17966, - "interval_diff": { - "value": 8069.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 9897 - } - ] - } - }, - { - "key": "Vehicle loan or lease", - "doc_count": 11, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1, - "interval_diff": { - "value": -5.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 6, - "interval_diff": { - "value": 2.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 4 - } - ] - } - }, - { - "key": "Student loan", - "doc_count": 8, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 3, - "interval_diff": { - "value": 3.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 0, - "interval_diff": { - "value": -5.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 5 - } - ] - } - }, - { - "key": "Mortgage", - "doc_count": 6, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1, - "interval_diff": { - "value": -1.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 2, - "interval_diff": { - "value": -1.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 3 - } - ] - } - }, - { - "key": "Credit card or prepaid card", - "doc_count": 5, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1, - "interval_diff": { - "value": 0.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 1, - "interval_diff": { - "value": -2.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 3 - } - ] - } - } - ] - } - }, - "max_date": { - "value": 1590512400000.0, - "value_as_string": "2020-05-26T12:00:00-05:00" - }, - "issue": { - "doc_count": 69251, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Incorrect information on your report", - "doc_count": 69251, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 15713, - "interval_diff": { - "value": -9935.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 25648, - "interval_diff": { - "value": 7672.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 17976, - "interval_diff": { - "value": 8062.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 9914 - } - ] - } - } - ] - } - }, - "min_date": { - "value": 1322758800000.0, - "value_as_string": "2011-12-01T12:00:00-05:00" - }, - "dateRangeArea": { - "doc_count": 69251, - "dateRangeArea": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 9914 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 17976 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 25648 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 15713 - } - ] - } - }, - "company": { - "doc_count": 69251, - "company": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "EQUIFAX, INC.", - "doc_count": 69251, - "trend_period": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 9914 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 17976, - "interval_diff": { - "value": 8062.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 25648, - "interval_diff": { - "value": 7672.0 - } - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 15713, - "interval_diff": { - "value": -9935.0 - } - } - ] - } - } - ] - } - }, - "sub-issue": { - "doc_count": 69251, - "sub-issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 3532, - "buckets": [ - { - "key": "Information belongs to someone else", - "doc_count": 43112, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 12212, - "interval_diff": { - "value": -5836.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 18048, - "interval_diff": { - "value": 8795.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 9253, - "interval_diff": { - "value": 5654.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 3599 - } - ] - } - }, - { - "key": "Account status incorrect", - "doc_count": 8419, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 1066, - "interval_diff": { - "value": -1171.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2237, - "interval_diff": { - "value": -869.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 3106, - "interval_diff": { - "value": 1096.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 2010 - } - ] - } - }, - { - "key": "Account information incorrect", - "doc_count": 8155, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 1157, - "interval_diff": { - "value": -1322.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2479, - "interval_diff": { - "value": -256.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 2735, - "interval_diff": { - "value": 951.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 1784 - } - ] - } - }, - { - "key": "Personal information incorrect", - "doc_count": 3535, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 590, - "interval_diff": { - "value": -641.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1231, - "interval_diff": { - "value": 203.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 1028, - "interval_diff": { - "value": 342.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 686 - } - ] - } - }, - { - "key": "Public record information inaccurate", - "doc_count": 2492, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 259, - "interval_diff": { - "value": -409.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 668, - "interval_diff": { - "value": -68.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 736, - "interval_diff": { - "value": -93.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 829 - } - ] - } - } - ] - } - }, - "tags": { - "doc_count": 69251, - "tags": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Servicemember", - "doc_count": 4626, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 658, - "interval_diff": { - "value": -786.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1444, - "interval_diff": { - "value": -51.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 1495, - "interval_diff": { - "value": 466.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 1029 - } - ] - } - }, - { - "key": "Older American", - "doc_count": 901, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 148, - "interval_diff": { - "value": -212.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 360, - "interval_diff": { - "value": 152.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 208, - "interval_diff": { - "value": 23.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 185 - } - ] - } - }, - { - "key": "Older American, Servicemember", - "doc_count": 283, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 34, - "interval_diff": { - "value": -67.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 101, - "interval_diff": { - "value": 26.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 75, - "interval_diff": { - "value": 2.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 73 - } - ] - } - } - ] - } - } - }, - "_meta": { - "date_min": "2011-12-01T12:00:00-05:00", - "date_max": "2020-05-26T12:00:00-05:00" - } -} \ No newline at end of file diff --git a/complaint_search/tests/expected_results/trends_sub_lens_product__valid.json b/complaint_search/tests/expected_results/trends_sub_lens_product__valid.json deleted file mode 100644 index 6f6eac41..00000000 --- a/complaint_search/tests/expected_results/trends_sub_lens_product__valid.json +++ /dev/null @@ -1,886 +0,0 @@ -{ - "took": 984, - "timed_out": false, - "_shards": { - "total": 5, - "successful": 5, - "failed": 0 - }, - "hits": { - "total": {"value": 1657420, "relation" : "gte"}, - "max_score": 0.0, - "hits": [] - }, - "aggregations": { - "dateRangeBrush": { - "doc_count": 1657420, - "dateRangeBrush": { - "buckets": [ - { - "key_as_string": "2011-01-01T00:00:00.000Z", - "key": 1293840000000, - "doc_count": 2536 - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 72373 - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 108217 - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 153043 - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 168475 - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 191468 - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 242965 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 257313 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 277390 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 183640 - } - ] - } - }, - "product": { - "doc_count": 1657420, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 388948, - "buckets": [ - { - "key": "Credit reporting, credit repair services, or other personal consumer reports", - "doc_count": 434259, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Credit reporting", - "doc_count": 427474, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 136459, - "interval_diff": { - "value": 26900.0 - } - } - ] - } - }, - { - "key": "Other personal consumer report", - "doc_count": 5419, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1619, - "interval_diff": { - "value": -85.0 - } - } - ] - } - }, - { - "key": "Credit repair services", - "doc_count": 1365, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 516, - "interval_diff": { - "value": 155.0 - } - } - ] - } - }, - { - "key": "Conventional home mortgage", - "doc_count": 1, - "trend_period": { - "buckets": [] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 110646, - "interval_diff": { - "value": -27948.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 138594, - "interval_diff": { - "value": 26969.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 111625, - "interval_diff": { - "value": 38231.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 73394 - } - ] - } - }, - { - "key": "Mortgage", - "doc_count": 304552, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 40665, - "buckets": [ - { - "key": "Other mortgage", - "doc_count": 86635, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 12942, - "interval_diff": { - "value": -382.0 - } - } - ] - } - }, - { - "key": "Conventional fixed mortgage", - "doc_count": 70613, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 13700, - "interval_diff": { - "value": -629.0 - } - } - ] - } - }, - { - "key": "Conventional home mortgage", - "doc_count": 45545, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 14043, - "interval_diff": { - "value": 19.0 - } - } - ] - } - }, - { - "key": "FHA mortgage", - "doc_count": 35714, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 3423, - "interval_diff": { - "value": -11.0 - } - } - ] - } - }, - { - "key": "Conventional adjustable mortgage (ARM)", - "doc_count": 25380, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 4638, - "interval_diff": { - "value": -493.0 - } - } - ] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 11136, - "interval_diff": { - "value": -11572.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 22708, - "interval_diff": { - "value": -1867.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 24575, - "interval_diff": { - "value": -6001.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 30576, - "interval_diff": { - "value": -10890.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 41466, - "interval_diff": { - "value": -879.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 42345, - "interval_diff": { - "value": -616.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 42961, - "interval_diff": { - "value": -6439.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 49400, - "interval_diff": { - "value": 11291.0 - } - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 38109, - "interval_diff": { - "value": 36833.0 - } - }, - { - "key_as_string": "2011-01-01T00:00:00.000Z", - "key": 1293840000000, - "doc_count": 1276 - } - ] - } - }, - { - "key": "Debt collection", - "doc_count": 300039, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 85359, - "buckets": [ - { - "key": "I do not know", - "doc_count": 62909, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 10142, - "interval_diff": { - "value": -577.0 - } - } - ] - } - }, - { - "key": "Other debt", - "doc_count": 45339, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 12645, - "interval_diff": { - "value": -2765.0 - } - } - ] - } - }, - { - "key": "Other (i.e. phone, health club, etc.)", - "doc_count": 44543, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 13062, - "interval_diff": { - "value": 1030.0 - } - } - ] - } - }, - { - "key": "Credit card debt", - "doc_count": 33191, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 10766, - "interval_diff": { - "value": -201.0 - } - } - ] - } - }, - { - "key": "Credit card", - "doc_count": 28698, - "trend_period": { - "buckets": [ - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 7353, - "interval_diff": { - "value": -520.0 - } - } - ] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 24101, - "interval_diff": { - "value": -22302.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 46403, - "interval_diff": { - "value": -4781.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 51184, - "interval_diff": { - "value": 3231.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 47953, - "interval_diff": { - "value": 7487.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 40466, - "interval_diff": { - "value": 742.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 39724, - "interval_diff": { - "value": 585.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 39139, - "interval_diff": { - "value": 28070.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 11069 - } - ] - } - }, - { - "key": "Credit reporting", - "doc_count": 140432, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 16587, - "interval_diff": { - "value": -27494.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 44081, - "interval_diff": { - "value": 9809.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 34272, - "interval_diff": { - "value": 5033.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 29239, - "interval_diff": { - "value": 14859.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 14380, - "interval_diff": { - "value": 12507.0 - } - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 1873 - } - ] - } - }, - { - "key": "Credit card", - "doc_count": 89190, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 7133, - "interval_diff": { - "value": -13932.0 - } - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 21065, - "interval_diff": { - "value": 3765.0 - } - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 17300, - "interval_diff": { - "value": 3326.0 - } - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 13974, - "interval_diff": { - "value": 869.0 - } - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 13105, - "interval_diff": { - "value": -2248.0 - } - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 15353, - "interval_diff": { - "value": 14093.0 - } - }, - { - "key_as_string": "2011-01-01T00:00:00.000Z", - "key": 1293840000000, - "doc_count": 1260 - } - ] - } - } - ] - } - }, - "max_date": { - "value": 1593622800000.0, - "value_as_string": "2020-07-01T12:00:00-05:00" - }, - "min_date": { - "value": 1322758800000.0, - "value_as_string": "2011-12-01T12:00:00-05:00" - }, - "dateRangeArea": { - "doc_count": 1657420, - "dateRangeArea": { - "buckets": [ - { - "key_as_string": "2011-01-01T00:00:00.000Z", - "key": 1293840000000, - "doc_count": 2536 - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 72373 - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 108217 - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 153043 - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 168475 - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 191468 - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 242965 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 257313 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 277390 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 183640 - } - ] - } - }, - "dateRangeBuckets": { - "doc_count": 1657420, - "dateRangeBuckets": { - "buckets": [ - { - "key_as_string": "2011-01-01T00:00:00.000Z", - "key": 1293840000000, - "doc_count": 2536 - }, - { - "key_as_string": "2012-01-01T00:00:00.000Z", - "key": 1325376000000, - "doc_count": 72373 - }, - { - "key_as_string": "2013-01-01T00:00:00.000Z", - "key": 1356998400000, - "doc_count": 108217 - }, - { - "key_as_string": "2014-01-01T00:00:00.000Z", - "key": 1388534400000, - "doc_count": 153043 - }, - { - "key_as_string": "2015-01-01T00:00:00.000Z", - "key": 1420070400000, - "doc_count": 168475 - }, - { - "key_as_string": "2016-01-01T00:00:00.000Z", - "key": 1451606400000, - "doc_count": 191468 - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 242965 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 257313 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 277390 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 183640 - } - ] - } - } - }, - "_meta": { - "date_min": "2011-12-01T12:00:00-05:00", - "date_max": "2020-07-01T12:00:00-05:00" - }, - "body": { - "from": 0, - "size": 0, - "_source": [ - "company", - "company_public_response", - "company_response", - "complaint_id", - "complaint_what_happened", - "date_received", - "date_sent_to_company", - "has_narrative", - "issue", - "product", - "state", - "submitted_via", - "sub_issue", - "sub_product", - "tags", - "timely", - "zip_code" - ], - "query": { - "query_string": { - "query": "*", - "fields": [ - "complaint_what_happened" - ], - "default_operator": "AND" - } - }, - "aggs": { - "dateRangeArea": { - "aggs": { - "dateRangeArea": { - "date_histogram": { - "field": "date_received", - "interval": "year" - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, - "dateRangeBrush": { - "aggs": { - "dateRangeBrush": { - "date_histogram": { - "field": "date_received", - "interval": "year" - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, - "product": { - "aggs": { - "product": { - "terms": { - "field": "product.raw", - "size": 5 - }, - "aggs": { - "trend_period": { - "date_histogram": { - "field": "date_received", - "interval": "year" - }, - "aggs": { - "interval_diff": { - "serial_diff": { - "buckets_path": "_count" - } - } - } - }, - "product": { - "terms": { - "field": "sub_product.raw", - "size": 5 - }, - "aggs": { - "trend_period": { - "date_histogram": { - "field": "date_received", - "interval": "year" - }, - "aggs": { - "interval_diff": { - "serial_diff": { - "buckets_path": "_count" - } - } - } - } - } - } - } - } - }, - "filter": { - "bool": { - "must": [], - "must_not": [] - } - } - }, - "min_date": { - "min": { - "field": "date_received", - "format": "yyyy-MM-dd'T'12:00:00-05:00" - } - }, - "max_date": { - "max": { - "field": "date_received", - "format": "yyyy-MM-dd'T'12:00:00-05:00" - } - } - } - } -} diff --git a/complaint_search/tests/expected_results/trends_top_self_filter__valid.json b/complaint_search/tests/expected_results/trends_top_self_filter__valid.json deleted file mode 100644 index 3b16c4b0..00000000 --- a/complaint_search/tests/expected_results/trends_top_self_filter__valid.json +++ /dev/null @@ -1,857 +0,0 @@ -{ - "took": 375, - "timed_out": false, - "_shards": { - "total": 5, - "successful": 5, - "failed": 0 - }, - "hits": { - "total": {"value": 1609587, "relation" : "gte"}, - "max_score": 0.0, - "hits": [] - }, - "aggregations": { - "dateRangeBrush": { - "doc_count": 256833, - "dateRangeBrush": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 37975 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 70244 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 90950 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 57664 - } - ] - } - }, - "product": { - "doc_count": 256833, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 881, - "buckets": [ - { - "key": "Credit reporting, credit repair services, or other personal consumer reports", - "doc_count": 248864, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Credit reporting", - "doc_count": 245948, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 87654, - "interval_diff": { - "value": 21238.0 - } - } - ] - } - }, - { - "key": "Other personal consumer report", - "doc_count": 2916, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 938, - "interval_diff": { - "value": -47.0 - } - } - ] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 56558, - "interval_diff": { - "value": -32034.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 88592, - "interval_diff": { - "value": 21191.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 67401, - "interval_diff": { - "value": 31088.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 36313 - } - ] - } - }, - { - "key": "Credit card or prepaid card", - "doc_count": 2689, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "General-purpose credit card or charge card", - "doc_count": 2019, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 610, - "interval_diff": { - "value": -58.0 - } - } - ] - } - }, - { - "key": "Store credit card", - "doc_count": 670, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 216, - "interval_diff": { - "value": -33.0 - } - } - ] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 419, - "interval_diff": { - "value": -407.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 826, - "interval_diff": { - "value": -91.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 917, - "interval_diff": { - "value": 390.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 527 - } - ] - } - }, - { - "key": "Vehicle loan or lease", - "doc_count": 1620, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Loan", - "doc_count": 1303, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 380, - "interval_diff": { - "value": -112.0 - } - } - ] - } - }, - { - "key": "Lease", - "doc_count": 308, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 106, - "interval_diff": { - "value": 3.0 - } - } - ] - } - }, - { - "key": "Title loan", - "doc_count": 9, - "trend_period": { - "buckets": [ - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 2 - } - ] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 236, - "interval_diff": { - "value": -257.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 493, - "interval_diff": { - "value": -104.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 597, - "interval_diff": { - "value": 303.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 294 - } - ] - } - }, - { - "key": "Mortgage", - "doc_count": 1492, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Conventional home mortgage", - "doc_count": 802, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 226, - "interval_diff": { - "value": -66.0 - } - } - ] - } - }, - { - "key": "FHA mortgage", - "doc_count": 286, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 82, - "interval_diff": { - "value": -14.0 - } - } - ] - } - }, - { - "key": "Other type of mortgage", - "doc_count": 192, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 46, - "interval_diff": { - "value": -16.0 - } - } - ] - } - }, - { - "key": "Home equity loan or line of credit (HELOC)", - "doc_count": 130, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 36, - "interval_diff": { - "value": -13.0 - } - } - ] - } - }, - { - "key": "VA mortgage", - "doc_count": 80, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 27, - "interval_diff": { - "value": 4.0 - } - } - ] - } - }, - { - "key": "Reverse mortgage", - "doc_count": 2, - "trend_period": { - "buckets": [] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 206, - "interval_diff": { - "value": -213.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 419, - "interval_diff": { - "value": -103.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 522, - "interval_diff": { - "value": 177.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 345 - } - ] - } - }, - { - "key": "Student loan", - "doc_count": 1287, - "product": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Federal student loan servicing", - "doc_count": 943, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 250, - "interval_diff": { - "value": -108.0 - } - } - ] - } - }, - { - "key": "Private student loan", - "doc_count": 344, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 86, - "interval_diff": { - "value": -47.0 - } - } - ] - } - } - ] - }, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 134, - "interval_diff": { - "value": -202.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 336, - "interval_diff": { - "value": -155.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 491, - "interval_diff": { - "value": 165.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 326 - } - ] - } - } - ] - } - }, - "max_date": { - "value": 1590512400000.0, - "value_as_string": "2020-05-26T12:00:00-05:00" - }, - "issue": { - "doc_count": 256833, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Incorrect information on your report", - "doc_count": 256833, - "trend_period": { - "buckets": [ - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 57664, - "interval_diff": { - "value": -33286.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 90950, - "interval_diff": { - "value": 20706.0 - } - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 70244, - "interval_diff": { - "value": 32269.0 - } - }, - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 37975 - } - ] - }, - "issue": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Information belongs to someone else", - "doc_count": 142692, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 56681, - "interval_diff": { - "value": 25205.0 - } - } - ] - } - }, - { - "key": "Account status incorrect", - "doc_count": 39929, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 11333, - "interval_diff": { - "value": -3107.0 - } - } - ] - } - }, - { - "key": "Account information incorrect", - "doc_count": 36563, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 11324, - "interval_diff": { - "value": -1148.0 - } - } - ] - } - }, - { - "key": "Personal information incorrect", - "doc_count": 11948, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 4276, - "interval_diff": { - "value": 833.0 - } - } - ] - } - }, - { - "key": "Old information reappears or never goes away", - "doc_count": 9426, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2465, - "interval_diff": { - "value": -731.0 - } - } - ] - } - }, - { - "key": "Public record information inaccurate", - "doc_count": 9382, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 2678, - "interval_diff": { - "value": -276.0 - } - } - ] - } - }, - { - "key": "Information is missing that should be on the report", - "doc_count": 4590, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1535, - "interval_diff": { - "value": 79.0 - } - } - ] - } - }, - { - "key": "Information is incorrect", - "doc_count": 692, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 197, - "interval_diff": { - "value": -52.0 - } - } - ] - } - }, - { - "key": "Information that should be on the report is missing", - "doc_count": 116, - "trend_period": { - "buckets": [ - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 41, - "interval_diff": { - "value": 5.0 - } - } - ] - } - }, - { - "key": "Incorrect information on your report", - "doc_count": 1, - "trend_period": { - "buckets": [] - } - } - ] - } - } - ] - } - }, - "collections": { - "doc_count": 256833, - "collections": { - "doc_count_error_upper_bound": 0, - "sum_other_doc_count": 0, - "buckets": [ - { - "key": "Servicemember", - "doc_count": 18870, - "trend_period": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 3920 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 6229, - "interval_diff": { - "value": 2309.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 5974, - "interval_diff": { - "value": -255.0 - } - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 2747, - "interval_diff": { - "value": -3227.0 - } - } - ] - } - }, - { - "key": "Older American", - "doc_count": 3819, - "trend_period": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 693 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 920, - "interval_diff": { - "value": 227.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 1566, - "interval_diff": { - "value": 646.0 - } - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 640, - "interval_diff": { - "value": -926.0 - } - } - ] - } - }, - { - "key": "Older American, Servicemember", - "doc_count": 1265, - "trend_period": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 260 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 342, - "interval_diff": { - "value": 82.0 - } - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 480, - "interval_diff": { - "value": 138.0 - } - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 183, - "interval_diff": { - "value": -297.0 - } - } - ] - } - } - ] - } - }, - "min_date": { - "value": 1322758800000.0, - "value_as_string": "2011-12-01T12:00:00-05:00" - }, - "dateRangeArea": { - "doc_count": 256833, - "dateRangeArea": { - "buckets": [ - { - "key_as_string": "2017-01-01T00:00:00.000Z", - "key": 1483228800000, - "doc_count": 37975 - }, - { - "key_as_string": "2018-01-01T00:00:00.000Z", - "key": 1514764800000, - "doc_count": 70244 - }, - { - "key_as_string": "2019-01-01T00:00:00.000Z", - "key": 1546300800000, - "doc_count": 90950 - }, - { - "key_as_string": "2020-01-01T00:00:00.000Z", - "key": 1577836800000, - "doc_count": 57664 - } - ] - } - } - }, - "_meta": { - "date_min": "2011-12-01T12:00:00-05:00", - "date_max": "2020-05-26T12:00:00-05:00" - } -} \ No newline at end of file diff --git a/complaint_search/tests/test_view_suggest_company.py b/complaint_search/tests/test_view_suggest_company.py index 8256e068..fed7b787 100644 --- a/complaint_search/tests/test_view_suggest_company.py +++ b/complaint_search/tests/test_view_suggest_company.py @@ -38,7 +38,7 @@ def test_suggest_text__valid(self, mock_essuggest): mock_essuggest.assert_called_once_with( "company.suggest", "company.raw", - field="complaint_what_happened", + field="all", format="default", frm=0, no_aggs=False, diff --git a/complaint_search/tests/test_views_suggest_company.py b/complaint_search/tests/test_views_suggest_company.py index 2a1f9995..4ae8e290 100644 --- a/complaint_search/tests/test_views_suggest_company.py +++ b/complaint_search/tests/test_views_suggest_company.py @@ -43,7 +43,7 @@ def test_suggest_text__valid(self, mock_essuggest): mock_essuggest.assert_called_once_with( "company.suggest", "company.raw", - field="complaint_what_happened", + field="all", format="default", frm=0, no_aggs=False, diff --git a/complaint_search/tests/test_views_suggest_zip.py b/complaint_search/tests/test_views_suggest_zip.py index 6238b07b..e0eb210c 100644 --- a/complaint_search/tests/test_views_suggest_zip.py +++ b/complaint_search/tests/test_views_suggest_zip.py @@ -38,7 +38,7 @@ def test_suggest_text__valid(self, mock_essuggest): mock_essuggest.assert_called_once_with( "zip_code", None, - field="complaint_what_happened", + field="all", format="default", frm=0, no_aggs=False, From 64f78b90ebfadb330de9664eb9ee62f6ee8c684b Mon Sep 17 00:00:00 2001 From: Wyatt Pearsall Date: Tue, 18 Aug 2026 10:31:02 -0700 Subject: [PATCH 5/6] update swagger docs --- swagger-config.yaml | 119 +------------------------------------------- 1 file changed, 2 insertions(+), 117 deletions(-) diff --git a/swagger-config.yaml b/swagger-config.yaml index 544987c0..566d0767 100644 --- a/swagger-config.yaml +++ b/swagger-config.yaml @@ -33,7 +33,6 @@ paths: - $ref: '#/components/parameters/company_response' - $ref: '#/components/parameters/date_received_max' - $ref: '#/components/parameters/date_received_min' - - $ref: '#/components/parameters/has_narrative' - $ref: '#/components/parameters/issue' - $ref: '#/components/parameters/product' - $ref: '#/components/parameters/search_after' @@ -54,24 +53,6 @@ paths: $ref: '#/components/schemas/SearchResult' '400': description: Invalid status value - /_suggest: - get: - tags: - - Typeahead - summary: Suggest possible searches - description: The endpoint for the main search box in the UI - parameters: - - $ref: '#/components/parameters/size' - - $ref: '#/components/parameters/suggest_text' - responses: - '200': - description: successful operation - content: - application/json: - schema: - $ref: '#/components/schemas/SuggestResult' - '400': - description: Invalid input /_suggest_company: get: tags: @@ -87,7 +68,6 @@ paths: - $ref: '#/components/parameters/company_response' - $ref: '#/components/parameters/date_received_max' - $ref: '#/components/parameters/date_received_min' - - $ref: '#/components/parameters/has_narrative' - $ref: '#/components/parameters/issue' - $ref: '#/components/parameters/product' - $ref: '#/components/parameters/state' @@ -119,7 +99,6 @@ paths: - $ref: '#/components/parameters/company_response' - $ref: '#/components/parameters/date_received_max' - $ref: '#/components/parameters/date_received_min' - - $ref: '#/components/parameters/has_narrative' - $ref: '#/components/parameters/issue' - $ref: '#/components/parameters/product' - $ref: '#/components/parameters/state' @@ -163,79 +142,6 @@ paths: description: Invalid ID supplied '404': description: Complaint not found - /geo/states: - get: - tags: - - Complaints - summary: Get the state-by-state information - description: Get complaint information broken down by states - parameters: - - $ref: '#/components/parameters/search_term' - - $ref: '#/components/parameters/field' - - $ref: '#/components/parameters/company' - - $ref: '#/components/parameters/company_public_response' - - $ref: '#/components/parameters/company_received_max' - - $ref: '#/components/parameters/company_received_min' - - $ref: '#/components/parameters/company_response' - - $ref: '#/components/parameters/date_received_max' - - $ref: '#/components/parameters/date_received_min' - - $ref: '#/components/parameters/has_narrative' - - $ref: '#/components/parameters/issue' - - $ref: '#/components/parameters/product' - - $ref: '#/components/parameters/state' - - $ref: '#/components/parameters/submitted_via' - - $ref: '#/components/parameters/tags' - - $ref: '#/components/parameters/timely' - - $ref: '#/components/parameters/zip_code' - responses: - '200': - description: successful operation - content: - application/json: - schema: - $ref: '#/components/schemas/StatesResult' - /trends: - get: - tags: - - Trends - summary: "List trends" - description: "Return specific aggregations for a search" - parameters: - - $ref: '#/components/parameters/search_term' - - $ref: '#/components/parameters/field' - - $ref: '#/components/parameters/company' - - $ref: '#/components/parameters/company_public_response' - - $ref: '#/components/parameters/company_received_max' - - $ref: '#/components/parameters/company_received_min' - - $ref: '#/components/parameters/company_response' - - $ref: '#/components/parameters/date_received_max' - - $ref: '#/components/parameters/date_received_min' - - $ref: '#/components/parameters/focus' - - $ref: '#/components/parameters/has_narrative' - - $ref: '#/components/parameters/issue' - - $ref: '#/components/parameters/lens' - - $ref: '#/components/parameters/product' - - $ref: '#/components/parameters/state' - - $ref: '#/components/parameters/submitted_via' - - $ref: '#/components/parameters/sub_lens' - - $ref: '#/components/parameters/sub_lens_depth' - - $ref: '#/components/parameters/tags' - - $ref: '#/components/parameters/timely' - - $ref: '#/components/parameters/trend_depth' - - $ref: '#/components/parameters/trend_interval' - - $ref: '#/components/parameters/zip_code' - responses: - '200': - description: successful operation - content: - application/json: - schema: - $ref: '#/components/schemas/TrendsResult' - text/csv: - schema: - $ref: '#/components/schemas/TrendsResult' - '400': - description: Invalid status value tags: - name: Complaints description: These endpoints provide access to consumer complaints @@ -308,14 +214,13 @@ components: field: name: field in: query - description: If the parameter "search_term" has a value, use "field" to specify which field is searched. If not specified, "complaint_what_happened" will be searched. + description: If the parameter "search_term" has a value, use "field" to specify which field is searched. If not specified, "all" will be searched. schema: type: string enum: - - complaint_what_happened - company_public_response - all - default: complaint_what_happened + default: all focus: name: focus in: query @@ -344,15 +249,6 @@ components: minimum: 1 maximum: 100000 default: 0 - has_narrative: - name: has_narrative - in: query - description: Filter the results to only return the specified state of whether it has narrative in the complaint or not, i.e. yes, no - explode: true - schema: - type: array - items: - type: string issue: name: issue in: query @@ -587,9 +483,6 @@ components: complaint_id: type: integer description: The unique identification number for a complaint - complaint_what_happened: - type: string - description: A description of the complaint provided by the consumer date_received: type: string format: date @@ -597,9 +490,6 @@ components: date_sent_to_company: type: string description: The date the CFPB sent the complaint to the company - has_narrative: - type: boolean - description: Indicates this complaint has a narrative issue: type: string description: The issue the consumer identified in the complaint @@ -666,9 +556,6 @@ components: is_data_stale: type: boolean description: Indicates the most recent data is over 5 business days old - is_narrative_stale: - type: boolean - description: Indicates the most recent narratives are over 5 business days old last_indexed: type: string description: The timestamp of the most recently indexed complaint @@ -730,8 +617,6 @@ components: $ref: '#/components/schemas/Aggregation' company_response: $ref: '#/components/schemas/Aggregation' - has_narrative: - $ref: '#/components/schemas/Aggregation' issue: $ref: '#/components/schemas/MultiLevelAggregation' product: From 1bec7b597b9e8b943aee5d5a5e5cf1085461f472 Mon Sep 17 00:00:00 2001 From: Richard Dinh <1038306+flacoman91@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:01:10 -0700 Subject: [PATCH 6/6] clean out remaining geo and trends work. --- .../states_date_filters__valid.json | 162 ------------------ complaint_search/tests/test_es_interface.py | 6 - swagger-config.yaml | 101 +---------- 3 files changed, 1 insertion(+), 268 deletions(-) delete mode 100644 complaint_search/tests/expected_results/states_date_filters__valid.json diff --git a/complaint_search/tests/expected_results/states_date_filters__valid.json b/complaint_search/tests/expected_results/states_date_filters__valid.json deleted file mode 100644 index 62371850..00000000 --- a/complaint_search/tests/expected_results/states_date_filters__valid.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "size": 0, - "track_total_hits": true, - "_source": [ - "company", - "company_public_response", - "company_response", - "complaint_id", - "date_received", - "date_sent_to_company", - "issue", - "product", - "state", - "submitted_via", - "sub_issue", - "sub_product", - "tags", - "timely", - "zip_code" - ], - "aggs": { - "issue": { - "filter": { - "bool": { - "must": [ - { - "range": { - "date_received": { - "from": "2020-01-01", - "to": "2020-05-05" - } - } - }, - { - "range": { - "date_sent_to_company": { - "from": "2020-01-01", - "to": "2020-05-05" - } - } - }, - { - "terms": { - "state": [ - "VA" - ] - } - } - ], - "must_not": [] - } - }, - "aggs": { - "issue": { - "terms": { - "field": "issue.raw", - "size": 5 - }, - "aggs": { - "issue": { - "terms": { - "field": "sub_issue.raw" - } - } - } - } - } - }, - "product": { - "filter": { - "bool": { - "must": [ - { - "range": { - "date_received": { - "from": "2020-01-01", - "to": "2020-05-05" - } - } - }, - { - "range": { - "date_sent_to_company": { - "from": "2020-01-01", - "to": "2020-05-05" - } - } - }, - { - "terms": { - "state": [ - "VA" - ] - } - } - ], - "must_not": [] - } - }, - "aggs": { - "product": { - "terms": { - "field": "product.raw", - "size": 5 - }, - "aggs": { - "product": { - "terms": { - "field": "sub_product.raw" - } - } - } - } - } - }, - "state": { - "filter": { - "bool": { - "must": [ - { - "range": { - "date_received": { - "from": "2020-01-01", - "to": "2020-05-05" - } - } - }, - { - "range": { - "date_sent_to_company": { - "from": "2020-01-01", - "to": "2020-05-05" - } - } - } - ], - "must_not": [] - } - }, - "aggs": { - "state": { - "terms": { - "field": "state", - "size": 100 - }, - "aggs": { - "product": { - "terms": { - "field": "product.raw" - } - }, - "issue": { - "terms": { - "field": "issue.raw" - } - } - } - } - } - } - } -} diff --git a/complaint_search/tests/test_es_interface.py b/complaint_search/tests/test_es_interface.py index bdfb688a..44ec42c2 100644 --- a/complaint_search/tests/test_es_interface.py +++ b/complaint_search/tests/test_es_interface.py @@ -91,12 +91,6 @@ def setUp(self): "aggregations": { "max_date": {"value_as_string": "2017-01-01"}, "max_indexed_date": {"value_as_string": "2017-01-02"}, - "max_narratives": { - "max_date": { - "value": 1483400000.0 - # 150970000.0 for November 3rd 2017 - } - }, } }, ] diff --git a/swagger-config.yaml b/swagger-config.yaml index 566d0767..c822a740 100644 --- a/swagger-config.yaml +++ b/swagger-config.yaml @@ -145,8 +145,6 @@ paths: tags: - name: Complaints description: These endpoints provide access to consumer complaints - - name: Trends - description: These endpoints provide access aggregated consumer complaint data - name: Typeahead description: These endpoints support the typeahead boxes in the UI externalDocs: @@ -218,18 +216,9 @@ components: schema: type: string enum: - - company_public_response + - company - all default: all - focus: - name: focus - in: query - description: The name of the product or company on which to focus charts for products and issues - explode: true - schema: - type: array - items: - type: string format: name: format in: query @@ -258,19 +247,6 @@ components: type: array items: type: string - lens: - name: lens - in: query - required: true - description: The data lens through which to view complaint trends over time. - schema: - type: string - enum: - - overview - - issue - - product - - tags - default: overview no_aggs: name: no_aggs in: query @@ -337,28 +313,6 @@ components: type: array items: type: string - sub_lens: - name: sub_lens - in: query - description: The sub-lens through which to view complaint trends over time. - schema: - type: string - enum: - - issue - - product - - sub_product - - sub_issue - - tags - sub_lens_depth: - name: sub_lens_depth - in: query - description: The top X trend sub aggregations will be returned, where X is the supplied sub_lens_depth. - schema: - type: integer - maximum: 10000000 - minimum: 5 - format: int64 - default: 10 submitted_via: name: submitted_via in: query @@ -393,29 +347,6 @@ components: type: array items: type: string - trend_depth: - name: trend_depth - in: query - description: The top X trend aggregations will be returned, where X is the supplied trend_depth. - schema: - type: integer - maximum: 10000000 - minimum: 5 - format: int64 - default: 10 - trend_interval: - name: trend_interval - in: query - description: The interval of time to use for trends aggregations histograms. When using day intervals, we recommend querying for date_received_min / max periods of less than one year. - required: true - schema: - type: string - enum: - - year - - quarter - - month - - week - - day zip_code: name: zip_code in: query @@ -633,40 +564,10 @@ components: $ref: '#/components/schemas/Aggregation' hits: $ref: '#/components/schemas/Hits' - StatesResult: - type: object - properties: - aggregations: - type: object - properties: - issue: - $ref: '#/components/schemas/MultiLevelAggregation' - product: - $ref: '#/components/schemas/MultiLevelAggregation' - state: - $ref: '#/components/schemas/MultiLevelAggregation' SuggestResult: type: array items: type: string - TrendsResult: - type: object - properties: - aggregations: - type: object - properties: - company: - $ref: '#/components/schemas/MultiLevelAggregation' - issue: - $ref: '#/components/schemas/MultiLevelAggregation' - product: - $ref: '#/components/schemas/MultiLevelAggregation' - sub_issue: - $ref: '#/components/schemas/MultiLevelAggregation' - sub_product: - $ref: '#/components/schemas/MultiLevelAggregation' - tags: - $ref: '#/components/schemas/MultiLevelAggregation' links: {} callbacks: {} security: []