Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/urlscan/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,5 @@
from .client import Client # noqa: F401
from .error import APIError, RateLimitError # noqa: F401
from .iterator import SearchIterator # noqa: F401
from .pro import Pro # noqa: F401
from .types import LiveScanResourceType, VisibilityType # noqa: F401
134 changes: 87 additions & 47 deletions src/urlscan/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ class RateLimitMemo(TypedDict):
RateLimitKey = Literal["public", "private", "unlisted", "retrieve", "search"]


class Client:
class BaseClient:
def __init__(
self,
api_key: str,
Expand Down Expand Up @@ -268,6 +268,44 @@ def post(
req = session.build_request("POST", path, json=json, data=data)
return self._send_request(session, req)

def put(
self,
path: str,
json: Any | None = None,
data: RequestData | None = None,
) -> ClientResponse:
"""Send a PUT request to a given API endpoint.

Args:
path (str): Path.
json (Any | None, optional): Dict to send in request body as JSON. Defaults to None.
data (RequestData | None, optional): Dict to send in request body. Defaults to None.

Returns:
ClientResponse: Response.
"""
session = self._get_session()
req = session.build_request("PUT", path, json=json, data=data)
return self._send_request(session, req)

def delete(
self,
path: str,
params: QueryParamTypes | None = None,
) -> ClientResponse:
"""Send a DELETE request to a given API endpoint.

Args:
path (str): Path.
params (QueryParamTypes | None, optional): Query parameters. Defaults to None.

Returns:
ClientResponse: Response.
"""
session = self._get_session()
req = session.build_request("DELETE", path, params=params)
return self._send_request(session, req)

def download(
self,
path: str,
Expand Down Expand Up @@ -296,6 +334,54 @@ def get_text(self, path: str, params: QueryParamTypes | None = None) -> str:
res = self.get(path, params=params)
return self._response_to_str(res)

def _get_error(self, res: ClientResponse) -> APIError | None:
try:
res.raise_for_status()
except httpx.HTTPStatusError as exc:
data: dict = exc.response.json()
message: str = data["message"]
description: str | None = data.get("description")
status: int = data["status"]

# ref. https://urlscan.io/docs/api/#ratelimit
if status == 429:
rate_limit_reset_after = float(
exc.response.headers.get("X-Rate-Limit-Reset-After", 0)
)
return RateLimitError(
message,
description=description,
status=status,
rate_limit_reset_after=rate_limit_reset_after,
)

return APIError(message, description=description, status=status)

return None

def _response_to_json(self, res: ClientResponse) -> dict:
error = self._get_error(res)
if error:
raise error

return res.json()

def _response_to_str(self, res: ClientResponse) -> str:
error = self._get_error(res)
if error:
raise error

return res.text

def _response_to_content(self, res: ClientResponse) -> bytes:
error = self._get_error(res)
if error:
raise error

return res.content


class Client(BaseClient):
def get_result(self, uuid: str) -> dict:
"""Get a result of a scan by UUID.

Expand Down Expand Up @@ -607,49 +693,3 @@ def mapping(res_or_error: dict | Exception) -> dict | Exception:
return self.get_result(uuid)

return [(url, mapping(res_or_error)) for url, res_or_error in responses]

def _get_error(self, res: ClientResponse) -> APIError | None:
try:
res.raise_for_status()
except httpx.HTTPStatusError as exc:
data: dict = exc.response.json()
message: str = data["message"]
description: str | None = data.get("description")
status: int = data["status"]

# ref. https://urlscan.io/docs/api/#ratelimit
if status == 429:
rate_limit_reset_after = float(
exc.response.headers.get("X-Rate-Limit-Reset-After", 0)
)
return RateLimitError(
message,
description=description,
status=status,
rate_limit_reset_after=rate_limit_reset_after,
)

return APIError(message, description=description, status=status)

return None

def _response_to_json(self, res: ClientResponse) -> dict:
error = self._get_error(res)
if error:
raise error

return res.json()

def _response_to_str(self, res: ClientResponse) -> str:
error = self._get_error(res)
if error:
raise error

return res.text

def _response_to_content(self, res: ClientResponse) -> bytes:
error = self._get_error(res)
if error:
raise error

return res.content
49 changes: 49 additions & 0 deletions src/urlscan/pro/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
from urlscan.client import BASE_URL, USER_AGENT, BaseClient, TimeoutTypes

from .livescan import LiveScan


class Pro(BaseClient):
def __init__(
self,
api_key: str,
base_url: str = BASE_URL,
user_agent: str = USER_AGENT,
trust_env: bool = False,
timeout: TimeoutTypes = 60,
proxy: str | None = None,
verify: bool = True,
retry: bool = False,
):
"""
Args:
api_key (str): Your urlscan.io API key.
base_url (str, optional): Base URL. Defaults to BASE_URL.
user_agent (str, optional): User agent. Defaults to USER_AGENT.
trust_env (bool, optional): Enable or disable usage of environment variables for configuration. Defaults to False.
timeout (TimeoutTypes, optional): timeout configuration to use when sending request. Defaults to 60.
proxy (str | None, optional): Proxy URL where all the traffic should be routed. Defaults to None.
verify (bool, optional): Either `True` to use an SSL context with the default CA bundle, `False` to disable verification. Defaults to True.
retry (bool, optional): Whether to use automatic X-Rate-Limit-Reset-After HTTP header based retry. Defaults to False.
"""
super().__init__(
api_key,
base_url=base_url,
user_agent=user_agent,
trust_env=trust_env,
timeout=timeout,
proxy=proxy,
verify=verify,
retry=retry,
)

self.livescan = LiveScan(
api_key=api_key,
base_url=base_url,
user_agent=user_agent,
trust_env=trust_env,
timeout=timeout,
proxy=proxy,
verify=verify,
retry=retry,
)
Loading