diff --git a/.ci/.matrix_exclude.yml b/.ci/.matrix_exclude.yml index 6d8878ab8..5e020b170 100644 --- a/.ci/.matrix_exclude.yml +++ b/.ci/.matrix_exclude.yml @@ -426,3 +426,12 @@ exclude: FRAMEWORK: httpx-0.14 - VERSION: python-3.14 FRAMEWORK: httpx-0.21 + # httpx2 requires Python 3.10+ + - VERSION: python-3.6 + FRAMEWORK: httpx2-newest + - VERSION: python-3.7 + FRAMEWORK: httpx2-newest + - VERSION: python-3.8 + FRAMEWORK: httpx2-newest + - VERSION: python-3.9 + FRAMEWORK: httpx2-newest diff --git a/.ci/.matrix_framework.yml b/.ci/.matrix_framework.yml index a47e12259..6ab5a5b60 100644 --- a/.ci/.matrix_framework.yml +++ b/.ci/.matrix_framework.yml @@ -50,6 +50,7 @@ FRAMEWORK: - pymemcache-newest - graphene-2 - httpx-newest + - httpx2-newest - httplib2-newest - prometheus_client-newest - sanic-newest diff --git a/.ci/.matrix_framework_fips.yml b/.ci/.matrix_framework_fips.yml index 6bbc9cd3e..51e00d885 100644 --- a/.ci/.matrix_framework_fips.yml +++ b/.ci/.matrix_framework_fips.yml @@ -18,6 +18,7 @@ FRAMEWORK: - starlette-newest - graphene-2 - httpx-newest + - httpx2-newest - httplib2-newest - prometheus_client-newest - sanic-newest diff --git a/.ci/.matrix_framework_full.yml b/.ci/.matrix_framework_full.yml index bf789b7d6..0751a2336 100644 --- a/.ci/.matrix_framework_full.yml +++ b/.ci/.matrix_framework_full.yml @@ -83,6 +83,7 @@ FRAMEWORK: - httpx-0.14 - httpx-0.21 - httpx-newest + - httpx2-newest - httplib2-newest - sanic-20.12 - sanic-newest diff --git a/docs/reference/supported-technologies.md b/docs/reference/supported-technologies.md index b31536390..62d043d97 100644 --- a/docs/reference/supported-technologies.md +++ b/docs/reference/supported-technologies.md @@ -578,6 +578,21 @@ Collected trace data: * requested URL +#### httpx2 [automatic-instrumentation-httpx2] + +Library: `httpx2` (`>=2`, requires Python 3.10+) + +Instrumented methods: + +* `httpx2.Client.send` +* `httpx2.AsyncClient.send` + +Collected trace data: + +* HTTP method +* requested URL + + ### Services [automatic-instrumentation-services] diff --git a/elasticapm/instrumentation/packages/httpx2/__init__.py b/elasticapm/instrumentation/packages/httpx2/__init__.py new file mode 100644 index 000000000..30599467c --- /dev/null +++ b/elasticapm/instrumentation/packages/httpx2/__init__.py @@ -0,0 +1,29 @@ +# BSD 3-Clause License +# +# Copyright (c) 2021, Elasticsearch BV +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/elasticapm/instrumentation/packages/httpx2/async/__init__.py b/elasticapm/instrumentation/packages/httpx2/async/__init__.py new file mode 100644 index 000000000..30599467c --- /dev/null +++ b/elasticapm/instrumentation/packages/httpx2/async/__init__.py @@ -0,0 +1,29 @@ +# BSD 3-Clause License +# +# Copyright (c) 2021, Elasticsearch BV +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/elasticapm/instrumentation/packages/httpx2/async/httpcore2.py b/elasticapm/instrumentation/packages/httpx2/async/httpcore2.py new file mode 100644 index 000000000..05a4c5ef7 --- /dev/null +++ b/elasticapm/instrumentation/packages/httpx2/async/httpcore2.py @@ -0,0 +1,102 @@ +# BSD 3-Clause License +# +# Copyright (c) 2021, Elasticsearch BV +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from elasticapm.contrib.asyncio.traces import async_capture_span +from elasticapm.instrumentation.packages.asyncio.base import AsyncAbstractInstrumentedModule +from elasticapm.instrumentation.packages.httpx import utils +from elasticapm.traces import DroppedSpan, execution_context +from elasticapm.utils import default_ports +from elasticapm.utils.disttracing import TracingOptions + + +class HTTPCore2AsyncInstrumentation(AsyncAbstractInstrumentedModule): + """ + This instrumentation only exists to make sure we add distributed tracing + headers on our requests from `httpx2`. `httpx2` is the only place this library + is used, so no spans will actually be created (due to already being in + a leaf span). However, the rest of the logic was left in (much of this + mirrors the urllib3 instrumentation) in case that situation ever changes. + """ + + name = "httpcore2" + + instrument_list = [ + ("httpcore2._async.connection", "AsyncHTTPConnection.handle_async_request"), + ] + + async def call(self, module, method, wrapped, instance, args, kwargs): + url, method, headers = utils.get_request_data(args, kwargs) + scheme, host, port, target = url + + if port is not None and port != default_ports.get(scheme): + host += ":" + str(port) + + signature = "%s %s" % (method.upper(), host) + + url = "%s://%s%s" % (scheme, host, target) + + transaction = execution_context.get_transaction() + + async with async_capture_span( + signature, + span_type="external", + span_subtype="http", + extra={"http": {"url": url}}, + leaf=True, + ) as span: + # if httpcore2 has been called in a leaf span, this span might be a DroppedSpan. + leaf_span = span + while isinstance(leaf_span, DroppedSpan): + leaf_span = leaf_span.parent + + if headers is not None: + # It's possible that there are only dropped spans, e.g. if we started dropping spans due to the + # transaction_max_spans limit. In this case, the transaction.id is used + parent_id = leaf_span.id if leaf_span else transaction.id + trace_parent = transaction.trace_parent.copy_from( + span_id=parent_id, trace_options=TracingOptions(recorded=True) + ) + utils.set_disttracing_headers(headers, trace_parent, transaction) + response = await wrapped(*args, **kwargs) + status_code = utils.get_status(response) + if status_code: + if span.context: + span.context["http"]["status_code"] = status_code + span.set_success() if status_code < 400 else span.set_failure() + return response + + def mutate_unsampled_call_args(self, module, method, wrapped, instance, args, kwargs, transaction): + # since we don't have a span, we set the span id to the transaction id + trace_parent = transaction.trace_parent.copy_from( + span_id=transaction.id, trace_options=TracingOptions(recorded=False) + ) + headers = utils.get_request_data(args, kwargs)[2] + utils.set_disttracing_headers(headers, trace_parent, transaction) + return args, kwargs diff --git a/elasticapm/instrumentation/packages/httpx2/async/httpx2.py b/elasticapm/instrumentation/packages/httpx2/async/httpx2.py new file mode 100644 index 000000000..8a6487906 --- /dev/null +++ b/elasticapm/instrumentation/packages/httpx2/async/httpx2.py @@ -0,0 +1,61 @@ +# BSD 3-Clause License +# +# Copyright (c) 2019, Elasticsearch BV +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from elasticapm.contrib.asyncio.traces import async_capture_span +from elasticapm.instrumentation.packages.asyncio.base import AsyncAbstractInstrumentedModule +from elasticapm.utils import get_host_from_url, sanitize_url + + +class Httpx2AsyncClientInstrumentation(AsyncAbstractInstrumentedModule): + name = "httpx2" + + instrument_list = [("httpx2", "AsyncClient.send")] + + async def call(self, module, method, wrapped, instance, args, kwargs): + request = kwargs.get("request") or args[0] + + request_method = request.method.upper() + url = str(request.url) + name = "{request_method} {host}".format(request_method=request_method, host=get_host_from_url(url)) + url = sanitize_url(url) + + async with async_capture_span( + name, + span_type="external", + span_subtype="http", + extra={"http": {"url": url}}, + leaf=True, + ) as span: + response = await wrapped(*args, **kwargs) + if response is not None: + if span.context: + span.context["http"]["status_code"] = response.status_code + span.set_success() if response.status_code < 400 else span.set_failure() + return response diff --git a/elasticapm/instrumentation/packages/httpx2/sync/__init__.py b/elasticapm/instrumentation/packages/httpx2/sync/__init__.py new file mode 100644 index 000000000..30599467c --- /dev/null +++ b/elasticapm/instrumentation/packages/httpx2/sync/__init__.py @@ -0,0 +1,29 @@ +# BSD 3-Clause License +# +# Copyright (c) 2021, Elasticsearch BV +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/elasticapm/instrumentation/packages/httpx2/sync/httpcore2.py b/elasticapm/instrumentation/packages/httpx2/sync/httpcore2.py new file mode 100644 index 000000000..ebea386b0 --- /dev/null +++ b/elasticapm/instrumentation/packages/httpx2/sync/httpcore2.py @@ -0,0 +1,94 @@ +# BSD 3-Clause License +# +# Copyright (c) 2021, Elasticsearch BV +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from elasticapm.instrumentation.packages.base import AbstractInstrumentedModule +from elasticapm.instrumentation.packages.httpx import utils +from elasticapm.traces import DroppedSpan, capture_span, execution_context +from elasticapm.utils import default_ports +from elasticapm.utils.disttracing import TracingOptions + + +class HTTPCore2Instrumentation(AbstractInstrumentedModule): + name = "httpcore2" + + instrument_list = [ + ("httpcore2._sync.connection", "HTTPConnection.handle_request"), + ] + + def call(self, module, method, wrapped, instance, args, kwargs): + url, method, headers = utils.get_request_data(args, kwargs) + scheme, host, port, target = url + if port is not None and port != default_ports.get(scheme): + host += ":" + str(port) + + signature = "%s %s" % (method.upper(), host) + + url = "%s://%s%s" % (scheme, host, target) + + transaction = execution_context.get_transaction() + + with capture_span( + signature, + span_type="external", + span_subtype="http", + extra={"http": {"url": url}}, + leaf=True, + ) as span: + # if httpcore2 has been called in a leaf span, this span might be a DroppedSpan. + leaf_span = span + while isinstance(leaf_span, DroppedSpan): + leaf_span = leaf_span.parent + + if headers is not None: + # It's possible that there are only dropped spans, e.g. if we started dropping spans. + # In this case, the transaction.id is used + parent_id = leaf_span.id if leaf_span else transaction.id + trace_parent = transaction.trace_parent.copy_from( + span_id=parent_id, trace_options=TracingOptions(recorded=True) + ) + utils.set_disttracing_headers(headers, trace_parent, transaction) + if leaf_span: + leaf_span.dist_tracing_propagated = True + response = wrapped(*args, **kwargs) + status_code = utils.get_status(response) + if status_code: + if span.context: + span.context["http"]["status_code"] = status_code + span.set_success() if status_code < 400 else span.set_failure() + return response + + def mutate_unsampled_call_args(self, module, method, wrapped, instance, args, kwargs, transaction): + # since we don't have a span, we set the span id to the transaction id + trace_parent = transaction.trace_parent.copy_from( + span_id=transaction.id, trace_options=TracingOptions(recorded=False) + ) + headers = utils.get_request_data(args, kwargs)[2] + utils.set_disttracing_headers(headers, trace_parent, transaction) + return args, kwargs diff --git a/elasticapm/instrumentation/packages/httpx2/sync/httpx2.py b/elasticapm/instrumentation/packages/httpx2/sync/httpx2.py new file mode 100644 index 000000000..1f3f16a41 --- /dev/null +++ b/elasticapm/instrumentation/packages/httpx2/sync/httpx2.py @@ -0,0 +1,61 @@ +# BSD 3-Clause License +# +# Copyright (c) 2019, Elasticsearch BV +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from elasticapm.instrumentation.packages.base import AbstractInstrumentedModule +from elasticapm.traces import capture_span +from elasticapm.utils import get_host_from_url, sanitize_url + + +class Httpx2ClientInstrumentation(AbstractInstrumentedModule): + name = "httpx2" + + instrument_list = [("httpx2", "Client.send")] + + def call(self, module, method, wrapped, instance, args, kwargs): + request = kwargs.get("request") or args[0] + + request_method = request.method.upper() + url = str(request.url) + name = "{request_method} {host}".format(request_method=request_method, host=get_host_from_url(url)) + url = sanitize_url(url) + + with capture_span( + name, + span_type="external", + span_subtype="http", + extra={"http": {"url": url}}, + leaf=True, + ) as span: + response = wrapped(*args, **kwargs) + if response is not None: + if span.context: + span.context["http"]["status_code"] = response.status_code + span.set_success() if response.status_code < 400 else span.set_failure() + return response diff --git a/elasticapm/instrumentation/register.py b/elasticapm/instrumentation/register.py index b37aff1e9..e480693ce 100644 --- a/elasticapm/instrumentation/register.py +++ b/elasticapm/instrumentation/register.py @@ -72,6 +72,16 @@ "elasticapm.instrumentation.packages.grpc.GRPCServerInstrumentation", } +if sys.version_info >= (3, 10): + _cls_register.update( + [ + "elasticapm.instrumentation.packages.httpx2.sync.httpx2.Httpx2ClientInstrumentation", + "elasticapm.instrumentation.packages.httpx2.sync.httpcore2.HTTPCore2Instrumentation", + "elasticapm.instrumentation.packages.httpx2.async.httpx2.Httpx2AsyncClientInstrumentation", + "elasticapm.instrumentation.packages.httpx2.async.httpcore2.HTTPCore2AsyncInstrumentation", + ] + ) + if sys.version_info >= (3, 7): _cls_register.update( [ diff --git a/setup.cfg b/setup.cfg index 0a56a9b01..8edffbf44 100644 --- a/setup.cfg +++ b/setup.cfg @@ -105,6 +105,7 @@ markers = starlette graphene httpx + httpx2 prometheus_client sanic asgi diff --git a/tests/instrumentation/asyncio_tests/httpx2_tests.py b/tests/instrumentation/asyncio_tests/httpx2_tests.py new file mode 100644 index 000000000..c2959d66a --- /dev/null +++ b/tests/instrumentation/asyncio_tests/httpx2_tests.py @@ -0,0 +1,213 @@ +# BSD 3-Clause License +# +# Copyright (c) 2019, Elasticsearch BV +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import pytest # isort:skip + +httpx2 = pytest.importorskip("httpx2") # isort:skip +httpcore2 = pytest.importorskip("httpcore2") # isort:skip + +import urllib.parse + +from elasticapm.conf import constants +from elasticapm.conf.constants import TRANSACTION +from elasticapm.contrib.asyncio.traces import async_capture_span +from elasticapm.utils.disttracing import TraceParent + +pytestmark = [pytest.mark.httpx2, pytest.mark.asyncio] + +allow_redirects = {"follow_redirects": False} + + +async def test_httpx2_instrumentation(instrument, elasticapm_client, waiting_httpserver): + waiting_httpserver.serve_content("") + url = waiting_httpserver.url + "/hello_world" + parsed_url = urllib.parse.urlparse(url) + elasticapm_client.begin_transaction("transaction.test") + async with async_capture_span("test_request", "test"): + async with httpx2.AsyncClient() as client: + await client.get(url, **allow_redirects) + elasticapm_client.end_transaction("MyView") + + transactions = elasticapm_client.events[TRANSACTION] + spans = elasticapm_client.spans_for_transaction(transactions[0]) + assert spans[0]["name"].startswith("GET 127.0.0.1:") + assert spans[0]["type"] == "external" + assert spans[0]["subtype"] == "http" + assert spans[0]["outcome"] == "success" + assert url == spans[0]["context"]["http"]["url"] + assert spans[0]["context"]["destination"]["service"] == { + "name": "", + "resource": "127.0.0.1:%d" % parsed_url.port, + "type": "", + } + + headers = waiting_httpserver.requests[0].headers + assert constants.TRACEPARENT_HEADER_NAME in headers + trace_parent = TraceParent.from_string( + headers[constants.TRACEPARENT_HEADER_NAME], tracestate_string=headers[constants.TRACESTATE_HEADER_NAME] + ) + assert trace_parent.trace_id == transactions[0]["trace_id"] + # Check that sample_rate was correctly placed in the tracestate + assert constants.TRACESTATE.SAMPLE_RATE in trace_parent.tracestate_dict + + # this should be the span id of `httpx2`, not of httpcore2 + assert trace_parent.span_id == spans[0]["id"] + assert trace_parent.trace_options.recorded + + +async def test_httpx2_instrumentation_string_url(instrument, elasticapm_client, waiting_httpserver): + waiting_httpserver.serve_content("") + url = waiting_httpserver.url + "/hello_world" + elasticapm_client.begin_transaction("transaction.test") + async with async_capture_span("test_request", "test"): + async with httpx2.AsyncClient() as client: + await client.get(url, **allow_redirects) + elasticapm_client.end_transaction("MyView") + + transactions = elasticapm_client.events[TRANSACTION] + spans = elasticapm_client.spans_for_transaction(transactions[0]) + assert spans[0]["name"].startswith("GET 127.0.0.1:") + assert url == spans[0]["context"]["http"]["url"] + + headers = waiting_httpserver.requests[0].headers + assert constants.TRACEPARENT_HEADER_NAME in headers + trace_parent = TraceParent.from_string( + headers[constants.TRACEPARENT_HEADER_NAME], tracestate_string=headers[constants.TRACESTATE_HEADER_NAME] + ) + assert trace_parent.trace_id == transactions[0]["trace_id"] + # Check that sample_rate was correctly placed in the tracestate + assert constants.TRACESTATE.SAMPLE_RATE in trace_parent.tracestate_dict + + # this should be the span id of `httpx2`, not of httpcore2 + assert trace_parent.span_id == spans[0]["id"] + assert trace_parent.trace_options.recorded + + +async def test_httpx2_instrumentation_malformed_empty(instrument, elasticapm_client): + from httpx2._exceptions import UnsupportedProtocol + + elasticapm_client.begin_transaction("transaction.test") + async with async_capture_span("test_request", "test"): + with pytest.raises(UnsupportedProtocol): + httpx2.get("") + + +async def test_httpx2_instrumentation_malformed_path(instrument, elasticapm_client): + from httpx2._exceptions import LocalProtocolError, UnsupportedProtocol + + elasticapm_client.begin_transaction("transaction.test") + async with async_capture_span("test_request", "test"): + async with httpx2.AsyncClient() as client: + # raises UnsupportedProtocol since 0.18.0 + with pytest.raises((LocalProtocolError, UnsupportedProtocol)): + await client.get("http://") + + +async def test_url_sanitization(instrument, elasticapm_client, waiting_httpserver): + waiting_httpserver.serve_content("") + url = waiting_httpserver.url + "/hello_world" + url = url.replace("http://", "http://user:pass@") + elasticapm_client.begin_transaction("transaction") + async with httpx2.AsyncClient() as client: + await client.get(url) + elasticapm_client.end_transaction("MyView") + transactions = elasticapm_client.events[TRANSACTION] + span = elasticapm_client.spans_for_transaction(transactions[0])[0] + + assert "pass" not in span["context"]["http"]["url"] + assert constants.MASK_URL in span["context"]["http"]["url"] + + +@pytest.mark.parametrize("status_code", [400, 500]) +async def test_httpx2_error(instrument, elasticapm_client, waiting_httpserver, status_code): + waiting_httpserver.serve_content("", code=status_code) + url = waiting_httpserver.url + "/hello_world" + parsed_url = urllib.parse.urlparse(url) + elasticapm_client.begin_transaction("transaction") + expected_sig = "GET {0}".format(parsed_url.netloc) + url = "http://{0}/hello_world".format(parsed_url.netloc) + async with async_capture_span("test_name", "test_type"): + async with httpx2.AsyncClient() as client: + await client.get(url, **allow_redirects) + + elasticapm_client.end_transaction("MyView") + + transactions = elasticapm_client.events[TRANSACTION] + spans = elasticapm_client.spans_for_transaction(transactions[0]) + + assert spans[0]["name"] == expected_sig + assert spans[0]["type"] == "external" + assert spans[0]["subtype"] == "http" + assert spans[0]["context"]["http"]["url"] == url + assert spans[0]["context"]["http"]["status_code"] == status_code + assert spans[0]["outcome"] == "failure" + + +async def test_httpx2_streaming(instrument, elasticapm_client, waiting_httpserver): + # client.stream passes the request as a keyword argument to the instrumented method. + # This helps test that we can handle it both as an arg and a kwarg + # see https://github.com/elastic/apm-agent-python/issues/1336 + elasticapm_client.begin_transaction("httpx2-context-manager-client-streaming") + client = httpx2.AsyncClient() + try: + async with client.stream("GET", url=waiting_httpserver.url) as response: + assert response + finally: + await client.aclose() + elasticapm_client.end_transaction("httpx2-context-manager-client-streaming") + + transactions = elasticapm_client.events[TRANSACTION] + span = elasticapm_client.spans_for_transaction(transactions[0])[0] + assert span["type"] == "external" + assert span["subtype"] == "http" + + +async def test_default_port_handling(instrument, elasticapm_client): + url = "https://example.com/" + + elasticapm_client.begin_transaction("transaction") + network_backend = httpcore2.AsyncMockBackend( + buffer=[ + b"HTTP/1.1 200 OK\r\n", + b"Content-Type: plain/text\r\n", + b"Content-Length: 13\r\n", + b"\r\n", + b"Hello, world!", + ] + ) + async with httpcore2.AsyncConnectionPool(network_backend=network_backend) as pool: + await pool.request("GET", url) + elasticapm_client.end_transaction("transaction") + + transactions = elasticapm_client.events[TRANSACTION] + span = elasticapm_client.spans_for_transaction(transactions[0])[0] + + assert span["name"] == "GET example.com" + assert span["context"]["http"]["url"] == url diff --git a/tests/instrumentation/httpx2_tests.py b/tests/instrumentation/httpx2_tests.py new file mode 100644 index 000000000..20b4ccb44 --- /dev/null +++ b/tests/instrumentation/httpx2_tests.py @@ -0,0 +1,209 @@ +# BSD 3-Clause License +# +# Copyright (c) 2019, Elasticsearch BV +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import pytest # isort:skip + +httpx2 = pytest.importorskip("httpx2") # isort:skip +httpcore2 = pytest.importorskip("httpcore2") # isort:skip + +import urllib.parse + +from elasticapm.conf import constants +from elasticapm.conf.constants import TRANSACTION +from elasticapm.traces import capture_span +from elasticapm.utils.disttracing import TraceParent + +pytestmark = pytest.mark.httpx2 + +allow_redirects = {"follow_redirects": False} + + +def test_httpx2_instrumentation(instrument, elasticapm_client, waiting_httpserver): + waiting_httpserver.serve_content("") + url = waiting_httpserver.url + "/hello_world" + parsed_url = urllib.parse.urlparse(url) + elasticapm_client.begin_transaction("transaction.test") + with capture_span("test_request", "test"): + httpx2.get(url, **allow_redirects) + elasticapm_client.end_transaction("MyView") + + transactions = elasticapm_client.events[TRANSACTION] + spans = elasticapm_client.spans_for_transaction(transactions[0]) + assert spans[0]["name"].startswith("GET 127.0.0.1:") + assert spans[0]["type"] == "external" + assert spans[0]["subtype"] == "http" + assert url == spans[0]["context"]["http"]["url"] + assert spans[0]["context"]["destination"]["service"] == { + "name": "", + "resource": "127.0.0.1:%d" % parsed_url.port, + "type": "", + } + assert spans[0]["context"]["service"]["target"]["type"] == "http" + assert spans[0]["context"]["service"]["target"]["name"] == f"127.0.0.1:{parsed_url.port}" + + headers = waiting_httpserver.requests[0].headers + assert constants.TRACEPARENT_HEADER_NAME in headers + trace_parent = TraceParent.from_string( + headers[constants.TRACEPARENT_HEADER_NAME], tracestate_string=headers[constants.TRACESTATE_HEADER_NAME] + ) + assert trace_parent.trace_id == transactions[0]["trace_id"] + # Check that sample_rate was correctly placed in the tracestate + assert constants.TRACESTATE.SAMPLE_RATE in trace_parent.tracestate_dict + + # this should be the span id of `httpx2`, not of httpcore2 + assert trace_parent.span_id == spans[0]["id"] + assert trace_parent.trace_options.recorded + + +def test_httpx2_instrumentation_via_client(instrument, elasticapm_client, waiting_httpserver): + waiting_httpserver.serve_content("") + url = waiting_httpserver.url + "/hello_world" + elasticapm_client.begin_transaction("transaction.test") + with capture_span("test_request", "test"): + c = httpx2.Client() + c.get(url, **allow_redirects) + elasticapm_client.end_transaction("MyView") + + transactions = elasticapm_client.events[TRANSACTION] + spans = elasticapm_client.spans_for_transaction(transactions[0]) + assert spans[0]["name"].startswith("GET 127.0.0.1:") + assert url == spans[0]["context"]["http"]["url"] + + headers = waiting_httpserver.requests[0].headers + assert constants.TRACEPARENT_HEADER_NAME in headers + trace_parent = TraceParent.from_string( + headers[constants.TRACEPARENT_HEADER_NAME], tracestate_string=headers[constants.TRACESTATE_HEADER_NAME] + ) + assert trace_parent.trace_id == transactions[0]["trace_id"] + # Check that sample_rate was correctly placed in the tracestate + assert constants.TRACESTATE.SAMPLE_RATE in trace_parent.tracestate_dict + + # this should be the span id of `httpx2`, not of httpcore2 + assert trace_parent.span_id == spans[0]["id"] + assert trace_parent.trace_options.recorded + + +def test_httpx2_instrumentation_malformed_empty(instrument, elasticapm_client): + from httpx2._exceptions import UnsupportedProtocol + + elasticapm_client.begin_transaction("transaction.test") + with capture_span("test_request", "test"): + with pytest.raises(UnsupportedProtocol): + httpx2.get("") + + +def test_httpx2_instrumentation_malformed_path(instrument, elasticapm_client): + from httpx2._exceptions import LocalProtocolError, UnsupportedProtocol + + elasticapm_client.begin_transaction("transaction.test") + with capture_span("test_request", "test"): + # raises UnsupportedProtocol since 0.18.0 + with pytest.raises((LocalProtocolError, UnsupportedProtocol)): + httpx2.get("http://") + + +def test_default_port_handling(instrument, elasticapm_client): + url = "https://example.com/" + elasticapm_client.begin_transaction("transaction") + network_backend = httpcore2.MockBackend( + buffer=[ + b"HTTP/1.1 200 OK\r\n", + b"Content-Type: plain/text\r\n", + b"Content-Length: 13\r\n", + b"\r\n", + b"Hello, world!", + ] + ) + with httpcore2.ConnectionPool(network_backend=network_backend) as pool: + pool.request("GET", url) + elasticapm_client.end_transaction("transaction") + transactions = elasticapm_client.events[TRANSACTION] + span = elasticapm_client.spans_for_transaction(transactions[0])[0] + + assert span["name"] == "GET example.com" + assert span["context"]["http"]["url"] == url + + +def test_url_sanitization(instrument, elasticapm_client, waiting_httpserver): + waiting_httpserver.serve_content("") + url = waiting_httpserver.url + "/hello_world" + url = url.replace("http://", "http://user:pass@") + elasticapm_client.begin_transaction("transaction") + httpx2.get(url) + elasticapm_client.end_transaction("MyView") + transactions = elasticapm_client.events[TRANSACTION] + span = elasticapm_client.spans_for_transaction(transactions[0])[0] + + assert "pass" not in span["context"]["http"]["url"] + assert constants.MASK_URL in span["context"]["http"]["url"] + + +@pytest.mark.parametrize("status_code", [400, 500]) +def test_httpx2_error(instrument, elasticapm_client, waiting_httpserver, status_code): + waiting_httpserver.serve_content("", code=status_code) + url = waiting_httpserver.url + "/hello_world" + parsed_url = urllib.parse.urlparse(url) + elasticapm_client.begin_transaction("transaction") + expected_sig = "GET {0}".format(parsed_url.netloc) + url = "http://{0}/hello_world".format(parsed_url.netloc) + with capture_span("test_name", "test_type"): + c = httpx2.Client() + c.get(url, **allow_redirects) + + elasticapm_client.end_transaction("MyView") + + transactions = elasticapm_client.events[TRANSACTION] + spans = elasticapm_client.spans_for_transaction(transactions[0]) + + assert spans[0]["name"] == expected_sig + assert spans[0]["type"] == "external" + assert spans[0]["subtype"] == "http" + assert spans[0]["context"]["http"]["url"] == url + assert spans[0]["context"]["http"]["status_code"] == status_code + assert spans[0]["outcome"] == "failure" + + +def test_httpx2_streaming(instrument, elasticapm_client, waiting_httpserver): + # client.stream passes the request as a keyword argument to the instrumented method. + # This helps test that we can handle it both as an arg and a kwarg + # see https://github.com/elastic/apm-agent-python/issues/1336 + elasticapm_client.begin_transaction("httpx2-context-manager-client-streaming") + client = httpx2.Client() + try: + with client.stream("GET", url=waiting_httpserver.url) as response: + assert response + finally: + client.close() + elasticapm_client.end_transaction("httpx2-context-manager-client-streaming") + + transactions = elasticapm_client.events[TRANSACTION] + span = elasticapm_client.spans_for_transaction(transactions[0])[0] + assert span["type"] == "external" + assert span["subtype"] == "http" diff --git a/tests/requirements/reqs-httpx2-newest.txt b/tests/requirements/reqs-httpx2-newest.txt new file mode 100644 index 000000000..199d1d67d --- /dev/null +++ b/tests/requirements/reqs-httpx2-newest.txt @@ -0,0 +1,2 @@ +httpx2 +-r reqs-base.txt diff --git a/tests/scripts/envs/httpx2.sh b/tests/scripts/envs/httpx2.sh new file mode 100644 index 000000000..4d40d8911 --- /dev/null +++ b/tests/scripts/envs/httpx2.sh @@ -0,0 +1 @@ +export PYTEST_MARKER="-m httpx2"