diff --git a/.env.template b/.env.template index ee0bd6b6..a0a00c22 100644 --- a/.env.template +++ b/.env.template @@ -14,7 +14,3 @@ OPEN_ROUTER_API_KEY= # Models VLM_PROVIDER_MODEL_ID= - -# Telemetry -ASKUI__VA__TELEMETRY__ENABLED=True # Set to "False" to disable telemetry - diff --git a/CLAUDE.md b/CLAUDE.md index 410f7d5f..06ef32e7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -194,7 +194,7 @@ Tools are auto-discovered and can be dynamically loaded via MCP configurations. - `src/askui/utils/` - Image processing, API utilities, caching, annotations - `src/askui/reporting.py` - Reporting & logging - `src/askui/retry.py` - Retry logic with exponential backoff -- `src/askui/telemetry/` - OpenTelemetry tracing & analytics +- `src/askui/telemetry/` - OpenTelemetry tracing ## Code Style & Conventions @@ -285,7 +285,6 @@ When writing or updating documentation in `docs/`: - Structured logging with structlog - Correlation IDs for request tracing - Prometheus metrics via FastAPI instrumentator -- Segment Analytics for usage tracking ## Extending the Framework diff --git a/docs/00_overview.md b/docs/00_overview.md index a048598a..3410030f 100644 --- a/docs/00_overview.md +++ b/docs/00_overview.md @@ -83,9 +83,9 @@ Access pre-built tools, integrate external services via MCP, and extend agent ca Generate human-readable logs of agent actions for debugging and auditing. ### 09 - Observability, Telemetry, Tracing -**Topics**: Telemetry data, privacy, disabling telemetry +**Topics**: OpenTelemetry tracing, OTLP export configuration -Understand what data is collected and how to opt out. +Export traces to your own observability backend. ### 10 - Extracting Data **Topics**: Using `get()`, file support (PDF. Excel, Word, CSV), structured data extraction, response schemas diff --git a/docs/09_observability_telemetry_tracing.md b/docs/09_observability_telemetry_tracing.md index 2d29d0db..835b459f 100644 --- a/docs/09_observability_telemetry_tracing.md +++ b/docs/09_observability_telemetry_tracing.md @@ -1,73 +1,5 @@ # Observability, Telemetry, and Tracing -Telemetry collects usage data to help improve AskUI Vision Agent. This data is used to detect bugs, understand usage patterns, and improve the user experience. - -### What Data Is Collected - -**Enabled by default**, telemetry records: - -- **Package Version**: `askui` package version in use -- **Environment Information**: - - Operating system and version - - System architecture (x86_64, arm64, etc.) - - Python version - - Device ID (hashed for privacy) -- **Session Information**: - - Session ID (random UUID per run) - - Workspace ID (if authenticated) - - User ID (if authenticated) -- **Method Calls**: - - Functions called (e.g., `click()`, `act()`, `get()`) - - Non-sensitive parameters (e.g., click coordinates) - - Execution duration -- **Exceptions**: - - Exception types and messages - - Stack traces (sanitized) - -### What Data Is NOT Collected - -- Screenshot content or image data -- Passwords or credentials -- User input text (what you type) -- Query text (what you ask the agent) -- Screen content or extracted data -- File paths or filenames -- Network traffic or API keys - -### Privacy Considerations - -- **Device ID Hashing**: Your machine's device ID is hashed with an AskUI-specific salt to prevent cross-application tracking -- **No PII**: No personally identifiable information is collected without explicit authentication -- **Workspace Linkage**: If `ASKUI_WORKSPACE_ID` and `ASKUI_TOKEN` are set, telemetry is linked to your workspace for better support -- **Open Source**: Telemetry code is open source and auditable in the repository - -### Disabling Telemetry - -To opt out of telemetry collection, set an environment variable: - -#### Linux & MacOS -```bash -export ASKUI__VA__TELEMETRY__ENABLED=False -``` - -#### Windows PowerShell -```powershell -$env:ASKUI__VA__TELEMETRY__ENABLED="False" -``` - -#### Windows Command Prompt -```cmd -set ASKUI__VA__TELEMETRY__ENABLED=False -``` - -#### In Code -```python -from askui import ComputerAgent -# Telemetry is now disabled for this session -``` - -When disabled, no usage data is collected or transmitted. - ## OpenTelemetry Tracing AskUI supports exporting traces via [OpenTelemetry](https://opentelemetry.io/) (OTLP/HTTP) for integration with observability backends like Grafana, Jaeger, or Datadog. diff --git a/docs/12_secrets.md b/docs/12_secrets.md index aad109f0..2582a2bf 100644 --- a/docs/12_secrets.md +++ b/docs/12_secrets.md @@ -77,7 +77,7 @@ with ComputerAgent(secrets=secrets) as agent: ## Supported Agents -`secrets=` is available on all agent types: `ComputerAgent`, `AndroidAgent`, `WebAgent`, `WebTestingAgent`, and `MultiDeviceAgent`. On `MultiDeviceAgent`, secrets are propagated to both the composed computer and Android agents. Secrets are excluded from telemetry. +`secrets=` is available on all agent types: `ComputerAgent`, `AndroidAgent`, `WebAgent`, `WebTestingAgent`, and `MultiDeviceAgent`. On `MultiDeviceAgent`, secrets are propagated to both the composed computer and Android agents. ## Limitations diff --git a/pdm.lock b/pdm.lock index b105f385..dd563ed1 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "all", "bedrock", "dev", "office-document", "otel", "vertex", "web"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:238adac982e7ce1a5e94921efaea67fb9c397102bd0da26158dea57110ef3887" +content_hash = "sha256:82aab08310294c91b62d7c2a1a34554b2873059f5a6fff05635e4b1227c92c4d" [[metadata.targets]] requires_python = ">=3.10,<3.14" @@ -248,17 +248,6 @@ files = [ {file = "authlib-1.7.2.tar.gz", hash = "sha256:2cea25fefcd4e7173bdf1372c0afc265c8034b23a8cd5dcb6a9164b826c64231"}, ] -[[package]] -name = "backoff" -version = "2.2.1" -requires_python = ">=3.7,<4.0" -summary = "Function decoration for backoff and retry" -groups = ["default"] -files = [ - {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, - {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, -] - [[package]] name = "backports-asyncio-runner" version = "1.2.0" @@ -3278,19 +3267,6 @@ files = [ {file = "py_key_value_aio-0.4.5.tar.gz", hash = "sha256:c6563a2c6abe5da5e20f4f9e875c2a9b425a2244a54fadbf46cf140a9eea45d7"}, ] -[[package]] -name = "py-machineid" -version = "1.0.0" -summary = "Get the unique machine ID of any host (without admin privileges)" -groups = ["default"] -dependencies = [ - "winregistry<3.0.0,>=2.0.1; sys_platform == \"win32\"", -] -files = [ - {file = "py_machineid-1.0.0-py3-none-any.whl", hash = "sha256:910df0d5f2663bcf6739d835c4949f4e9cc6bb090a58b3dd766e12e5f768e3b9"}, - {file = "py_machineid-1.0.0.tar.gz", hash = "sha256:8a902a00fae8c6d6433f463697c21dc4ce98c6e55a2e0535c0273319acb0047a"}, -] - [[package]] name = "py-serializable" version = "2.1.0" @@ -3674,7 +3650,7 @@ name = "python-dateutil" version = "2.9.0.post0" requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" summary = "Extensions to the standard Python datetime module" -groups = ["default", "all", "bedrock", "dev", "office-document", "vertex"] +groups = ["all", "bedrock", "dev", "office-document", "vertex"] dependencies = [ "six>=1.5", ] @@ -4190,23 +4166,6 @@ files = [ {file = "secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be"}, ] -[[package]] -name = "segment-analytics-python" -version = "2.3.6" -requires_python = ">=3.9.0" -summary = "The hassle-free way to integrate analytics into any python application." -groups = ["default"] -dependencies = [ - "PyJWT~=2.12", - "backoff~=2.1", - "python-dateutil~=2.2", - "requests~=2.7", -] -files = [ - {file = "segment_analytics_python-2.3.6-py2.py3-none-any.whl", hash = "sha256:a8c5c0f542be3be39aa57cd77e7e6b85b2a3c348174bc1192a296aeb81626594"}, - {file = "segment_analytics_python-2.3.6.tar.gz", hash = "sha256:b07779d90432399fdd08f8cbeb483139cab862b5dca1bb3f0711f415def776a8"}, -] - [[package]] name = "setuptools" version = "82.0.1" @@ -4223,7 +4182,7 @@ name = "six" version = "1.17.0" requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" summary = "Python 2 and 3 compatibility utilities" -groups = ["default", "all", "bedrock", "dev", "office-document", "vertex"] +groups = ["all", "bedrock", "dev", "office-document", "vertex"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -4672,18 +4631,6 @@ files = [ {file = "websockets-16.0.tar.gz", hash = "sha256:5f6261a5e56e8d5c42a4497b364ea24d94d9563e8fbd44e78ac40879c60179b5"}, ] -[[package]] -name = "winregistry" -version = "2.1.5" -requires_python = ">=3.8" -summary = "A Python library for interacting with the Windows registry" -groups = ["default"] -marker = "sys_platform == \"win32\"" -files = [ - {file = "winregistry-2.1.5-py3-none-any.whl", hash = "sha256:84e597b8f06c985a6be397e8c75400c2d72797d24134eb740b7b2f3de13089c1"}, - {file = "winregistry-2.1.5.tar.gz", hash = "sha256:05525ecac026dbdd2d20428a532354138dc6711de8a8f8efdc40484131627ba8"}, -] - [[package]] name = "wrapt" version = "2.2.2" diff --git a/pyproject.toml b/pyproject.toml index 4890ef4b..853abcbc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,11 +16,9 @@ dependencies = [ "openai>=1.61.1", "opentelemetry-sdk>=1.38.0", "pillow>=11.0.0", - "py-machineid>=0.7.0", "pydantic-settings>=2.9.1", "pydantic>=2.11.0", "pyperclip>=1.9.0", - "segment-analytics-python>=2.3.4", "tenacity>=9.1.2", "jsonref>=1.1.0", "protobuf>=6.31.1", @@ -186,7 +184,6 @@ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" "src/askui/models/shared/android_agent.py" = ["E501"] "src/askui/prompts/system.py" = ["E501"] "src/askui/reporting.py" = ["E501"] -"src/askui/telemetry/telemetry.py" = ["E501"] "src/askui/tools/askui/askui_ui_controller_grpc/*" = ["ALL"] "src/askui/utils/image_utils.py" = ["E501"] "src/askui/web_agent.py" = ["E501"] diff --git a/src/askui/agent_base.py b/src/askui/agent_base.py index 84df7198..30b42ad3 100644 --- a/src/askui/agent_base.py +++ b/src/askui/agent_base.py @@ -11,7 +11,6 @@ from askui.agent_settings import AgentSettings from askui.callbacks import ConversationCallback, ConversationStatisticsCallback -from askui.container import telemetry from askui.locators.locators import Locator from askui.models.shared.agent_message_param import MessageParam, TextBlockParam from askui.models.shared.conversation import Conversation, Speakers @@ -128,9 +127,6 @@ def __init__( self.locate_settings = LocateSettings() self.caching_settings = CachingSettings() - @telemetry.record_call( - exclude={"goal", "act_settings", "tools", "tracing_settings", "secrets"} - ) @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def act( self, @@ -625,9 +621,6 @@ def get( get_settings: GetSettings | None = None, ) -> ResponseSchema: ... - @telemetry.record_call( - exclude={"query", "source", "response_schema", "get_settings"} - ) @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def get( self, @@ -813,7 +806,6 @@ def locate_with_screenshot() -> PointList: logger.debug("LocateModel locate: %d elements", len(points)) return points - @telemetry.record_call(exclude={"locator", "screenshot", "locate_settings"}) @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def locate( self, @@ -859,7 +851,6 @@ def locate( locate_settings=locate_settings, )[0] - @telemetry.record_call(exclude={"locator", "screenshot", "locate_settings"}) @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def locate_all( self, @@ -908,7 +899,6 @@ def locate_all( locate_settings=locate_settings, ) - @telemetry.record_call(exclude={"screenshot"}) @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def locate_all_elements( self, @@ -947,7 +937,6 @@ def locate_all_elements( locate_settings=self.locate_settings, ) - @telemetry.record_call(exclude={"screenshot", "annotation_dir"}) @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def annotate( self, @@ -1011,7 +1000,6 @@ def annotate( "AnnotationWriter", f"annotated HTML file saved to '{annotated_html}'" ) - @telemetry.record_call(exclude={"until"}) @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def wait( self, @@ -1136,23 +1124,19 @@ def _wait_for_disappear( ) raise WaitUntilError(locator, str(locator), retry_count, delay, "disappear") - @telemetry.record_call() def close(self) -> None: if self._agent_os is not None: self._agent_os.disconnect() self._reporter.generate() - @telemetry.record_call() def open(self) -> None: if self._agent_os is not None: self._agent_os.connect() - @telemetry.record_call() def __enter__(self) -> Self: self.open() return self - @telemetry.record_call(exclude={"exc_value", "traceback"}) def __exit__( self, exc_type: Type[BaseException] | None, diff --git a/src/askui/android_agent.py b/src/askui/android_agent.py index 916b78ec..03c06d76 100644 --- a/src/askui/android_agent.py +++ b/src/askui/android_agent.py @@ -7,7 +7,6 @@ from askui.agent_base import Agent from askui.agent_settings import AgentSettings from askui.callbacks import ConversationCallback -from askui.container import telemetry from askui.locators.locators import Locator from askui.models.models import Point from askui.models.shared.secrets import Secret @@ -70,16 +69,6 @@ class AndroidAgent(Agent): ``` """ - @telemetry.record_call( - exclude={ - "reporters", - "settings", - "act_tools", - "callbacks", - "truncation_strategy", - "secrets", - } - ) @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def __init__( self, @@ -139,7 +128,6 @@ def tap( target: Point, ) -> None: ... - @telemetry.record_call(exclude={"locator"}) @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def tap( self, @@ -174,7 +162,6 @@ def tap( point = self._locate(locator=target)[0] self.os.tap(point[0], point[1]) - @telemetry.record_call(exclude={"text"}) @validate_call def type( self, @@ -202,7 +189,6 @@ def type( logger.debug("AndroidAgent received instruction to type") self.os.type(self._resolve_secrets(text)) - @telemetry.record_call() @validate_call def key_tap( self, @@ -225,7 +211,6 @@ def key_tap( """ self.os.key_tap(key) - @telemetry.record_call() @validate_call def key_combination( self, @@ -254,7 +239,6 @@ def key_combination( ) self.os.key_combination(keys, duration_in_ms) - @telemetry.record_call() @validate_call def shell( self, @@ -278,7 +262,6 @@ def shell( self._reporter.add_message("User", f"shell(command='{command}')") return self.os.shell(command) - @telemetry.record_call() @validate_call def drag_and_drop( self, @@ -312,7 +295,6 @@ def drag_and_drop( ) self.os.drag_and_drop(x1, y1, x2, y2, duration_in_ms) - @telemetry.record_call() @validate_call def swipe( self, @@ -346,9 +328,6 @@ def swipe( ) self.os.swipe(x1, y1, x2, y2, duration_in_ms) - @telemetry.record_call( - exclude={"device_sn"}, - ) @validate_call def set_device_by_serial_number( self, diff --git a/src/askui/computer_agent.py b/src/askui/computer_agent.py index 8cf70bf1..36962975 100644 --- a/src/askui/computer_agent.py +++ b/src/askui/computer_agent.py @@ -7,7 +7,6 @@ from askui.agent_base import Agent from askui.agent_settings import AgentSettings from askui.callbacks import ConversationCallback -from askui.container import telemetry from askui.locators.locators import Locator from askui.models.models import Point from askui.models.shared.secrets import Secret @@ -155,17 +154,6 @@ class ComputerAgent(Agent): ``` """ - @telemetry.record_call( - exclude={ - "reporters", - "settings", - "act_tools", - "callbacks", - "truncation_strategy", - "agent_os_target_computers", - "secrets", - } - ) @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def __init__( self, @@ -211,7 +199,6 @@ def __init__( ) ) - @telemetry.record_call(exclude={"locator"}) @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def click( self, @@ -288,7 +275,6 @@ def _mouse_move( point = (point[0] + offset[0], point[1] + offset[1]) self.tools.os.mouse_move(point[0], point[1]) - @telemetry.record_call(exclude={"locator"}) @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def mouse_move( self, @@ -319,7 +305,6 @@ def mouse_move( logger.debug("Agent received instruction to mouse_move to %s", locator) self._mouse_move(locator, offset, locate_settings) - @telemetry.record_call() @validate_call def mouse_scroll( self, @@ -353,7 +338,6 @@ def mouse_scroll( self._reporter.add_message("User", f'mouse_scroll: "{dx}", "{dy}"') self.tools.os.mouse_scroll(dx, dy) - @telemetry.record_call(exclude={"text", "locator"}) @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def type( self, @@ -411,7 +395,6 @@ def type( self._reporter.add_message("User", msg) self.tools.os.type(self._resolve_secrets(text)) - @telemetry.record_call() @validate_call def key_up( self, @@ -436,7 +419,6 @@ def key_up( logger.debug("ComputerAgent received in key_up '%s'", key) self.tools.os.keyboard_release(key) - @telemetry.record_call() @validate_call def key_down( self, @@ -461,7 +443,6 @@ def key_down( logger.debug("ComputerAgent received in key_down '%s'", key) self.tools.os.keyboard_pressed(key) - @telemetry.record_call() @validate_call def mouse_up( self, @@ -487,7 +468,6 @@ def mouse_up( logger.debug("ComputerAgent received instruction to mouse_up '%s'", button) self.tools.os.mouse_up(button) - @telemetry.record_call() @validate_call def mouse_down( self, @@ -513,7 +493,6 @@ def mouse_down( logger.debug("ComputerAgent received instruction to mouse_down '%s'", button) self.tools.os.mouse_down(button) - @telemetry.record_call() @validate_call def keyboard( self, @@ -551,7 +530,6 @@ def keyboard( logger.debug("ComputerAgent received instruction to press '%s'", key) self.tools.os.keyboard_tap(key, modifier_keys, count=repeat) - @telemetry.record_call(exclude={"command"}) @validate_call def cli( self, diff --git a/src/askui/container.py b/src/askui/container.py deleted file mode 100644 index 765cc904..00000000 --- a/src/askui/container.py +++ /dev/null @@ -1,8 +0,0 @@ -from askui.settings import SETTINGS -from askui.telemetry import Telemetry -from askui.telemetry.processors import Segment - -telemetry = Telemetry(SETTINGS.telemetry) - -if SETTINGS.telemetry.segment: - telemetry.add_processor(Segment(SETTINGS.telemetry.segment)) diff --git a/src/askui/models/models.py b/src/askui/models/models.py index 993f4b2a..c9cc584b 100644 --- a/src/askui/models/models.py +++ b/src/askui/models/models.py @@ -190,32 +190,6 @@ def act( """ # noqa: E501 raise NotImplementedError - def to_telemetry_dict(self) -> dict[str, str]: - """Return a JSON-serializable representation for telemetry. - - For ActModels, this includes the model_id and MessagesApi class name. - - Returns: - dict[str, str]: A dictionary with telemetry information about this model. - Includes: "type", "class", "model_id", "messages_api". - - Example: - ```python - { - "type": "act_model", - "class": "AskUIAgent", - "model_id": "claude-sonnet-4-20250514", - "messages_api": "AnthropicMessagesApi" - } - ``` - """ - return { - "type": "act_model", - "class": self.__class__.__name__, - "model_id": self.__getattribute__("_model_id" or "-"), - "messages_api": self._messages_api.__class__.__name__, # type: ignore[attr-defined] - } - class GetModel(abc.ABC): """Abstract base class for models that can extract information from images and PDFs. @@ -268,28 +242,6 @@ def get( """ raise NotImplementedError - def to_telemetry_dict(self) -> dict[str, str]: - """Return a JSON-serializable representation for telemetry. - - For GetModels, this includes the class name. - - Returns: - dict[str, str]: A dictionary with telemetry information about this model. - Includes: "type", "class". - - Example: - ```python - { - "type": "get_model", - "class": "AskUiGeminiGetModel" - } - ``` - """ - return { - "type": "get_model", - "class": self.__class__.__name__, - } - class LocateModel(abc.ABC): """Abstract base class for models that can locate UI elements in images. @@ -355,28 +307,6 @@ def locate_all_elements( """ raise NotImplementedError - def to_telemetry_dict(self) -> dict[str, str]: - """Return a JSON-serializable representation for telemetry. - - For LocateModels, this includes the class name. - - Returns: - dict[str, str]: A dictionary with telemetry information about this model. - Includes: "type", "class". - - Example: - ```python - { - "type": "locate_model", - "class": "AskUiPtaLocateModel" - } - ``` - """ - return { - "type": "locate_model", - "class": self.__class__.__name__, - } - Model = ActModel | GetModel | LocateModel """Union type of all abstract model classes. diff --git a/src/askui/settings.py b/src/askui/settings.py deleted file mode 100644 index aa317e22..00000000 --- a/src/askui/settings.py +++ /dev/null @@ -1,22 +0,0 @@ -from pydantic import Field -from pydantic_settings import BaseSettings, SettingsConfigDict - -from askui.telemetry import TelemetrySettings - - -class Settings(BaseSettings): - """Main settings class""" - - model_config = SettingsConfigDict( - env_prefix="ASKUI__VA__", - env_file=".env", - env_file_encoding="utf-8", - case_sensitive=False, - env_nested_delimiter="__", - extra="ignore", - ) - - telemetry: TelemetrySettings = Field(default_factory=TelemetrySettings) - - -SETTINGS = Settings() diff --git a/src/askui/telemetry/__init__.py b/src/askui/telemetry/__init__.py index 60ad44ea..e69de29b 100644 --- a/src/askui/telemetry/__init__.py +++ b/src/askui/telemetry/__init__.py @@ -1,23 +0,0 @@ -from .context import ( - AppContext, - DeviceContext, - OSContext, - PlatformContext, - TelemetryContext, -) -from .processors import InMemoryProcessor, Segment, TelemetryEvent, TelemetryProcessor -from .telemetry import Telemetry, TelemetrySettings - -__all__ = [ - "AppContext", - "DeviceContext", - "InMemoryProcessor", - "OSContext", - "PlatformContext", - "Segment", - "Telemetry", - "TelemetryContext", - "TelemetryEvent", - "TelemetryProcessor", - "TelemetrySettings", -] diff --git a/src/askui/telemetry/anonymous_id.py b/src/askui/telemetry/anonymous_id.py deleted file mode 100644 index 03b92b50..00000000 --- a/src/askui/telemetry/anonymous_id.py +++ /dev/null @@ -1,64 +0,0 @@ -import logging -import uuid -from pathlib import Path - -from askui.telemetry.device_id import get_device_id -from askui.telemetry.utils import hash_to_uuid4, is_valid_uuid4 - -logger = logging.getLogger(__name__) - -_ANONYMOUS_ID_FILE_PATH = Path.home() / ".askui" / "ADK" / "anonymous_id" -_anonymous_id: str | None = None - - -def _read_anonymous_id_from_file() -> str | None: - """Read anonymous ID from file if it exists.""" - try: - if _ANONYMOUS_ID_FILE_PATH.exists(): - return _ANONYMOUS_ID_FILE_PATH.read_text(encoding="utf-8").strip() - except OSError as e: - logger.warning("Failed to read anonymous ID from file", extra={"error": str(e)}) - return None - - -def _write_anonymous_id_to_file(anonymous_id: str) -> bool: - """Write anonymous ID to file, creating directories if needed.""" - try: - _ANONYMOUS_ID_FILE_PATH.parent.mkdir(parents=True, exist_ok=True) - _ANONYMOUS_ID_FILE_PATH.write_text(anonymous_id, encoding="utf-8") - except OSError as e: - logger.warning("Failed to write anonymous ID to file", extra={"error": str(e)}) - else: - return True - return False - - -def get_anonymous_id() -> str: - """Get an anonymous (user) ID for telemetry purposes. - - Returns: - str: A UUID v4 string in lowercase format. - - The function follows this process: - 1. Returns cached ID if available in memory - 2. Attempts to read ID from disk (`~/.askui/ADK/anonymous_id`) if not in memory - 3. If ID doesn't exist or is invalid, generates a new one: - - Derived from device ID if available - - Random UUID if device ID unavailable - 4. Writes new ID to disk for persistence and returns it - 5. If writing to disk fails, just returns the new ID for each run - - Only going to be same across runs if it can be derived from the device ID, - otherwise it's random - """ - global _anonymous_id - if _anonymous_id is None: - aid = _read_anonymous_id_from_file() - if aid is None or not is_valid_uuid4(aid): - machine_id = get_device_id() - if machine_id: - aid = hash_to_uuid4(machine_id).lower() - else: - aid = str(uuid.uuid4()).lower() - _write_anonymous_id_to_file(aid) - _anonymous_id = aid - return _anonymous_id diff --git a/src/askui/telemetry/context.py b/src/askui/telemetry/context.py deleted file mode 100644 index b4b01cb0..00000000 --- a/src/askui/telemetry/context.py +++ /dev/null @@ -1,81 +0,0 @@ -import os -from typing import TypedDict - -from pydantic import ConfigDict -from typing_extensions import NotRequired - - -class AppContext(TypedDict, total=False): - """App (askui SDK/library/package) context information.""" - - __pydantic_config__ = ConfigDict(extra="allow") # type: ignore - - name: str - version: str - - -class OSContext(TypedDict, total=False): - """OS context information.""" - - __pydantic_config__ = ConfigDict(extra="allow") # type: ignore - - name: str - version: str - release: str - - -class PlatformContext(TypedDict, total=False): - """Platform context information.""" - - __pydantic_config__ = ConfigDict(extra="allow") # type: ignore - - arch: str - python_version: str - - -class DeviceContext(TypedDict, total=False): - """Device context information.""" - - __pydantic_config__ = ConfigDict(extra="allow") # type: ignore - - id: str - - -class TelemetryContext(TypedDict, total=False): - """Context information.""" - - __pydantic_config__ = ConfigDict(extra="allow") # type: ignore - - app: AppContext - user_id: NotRequired[str] - anonymous_id: str - group_id: NotRequired[str] - os: OSContext - platform: PlatformContext - device: NotRequired[DeviceContext] - session_id: str - call_stack: list[str] - - -class CallStack: - def __init__(self, max_depth: int = 100) -> None: - self._stack: list[str] = [] - self._max_depth = max_depth - - @property - def current(self) -> list[str]: - return self._stack - - def push_call(self) -> str: - """Generates call id and adds it to the call stack and returns it""" - if len(self._stack) >= self._max_depth: - error_msg = "Call stack is at max depth" - raise ValueError(error_msg) - - call_id = os.urandom(8).hex() - self._stack.append(call_id) - return call_id - - def pop_call(self) -> str: - """Removes the last call from the call stack and returns it""" - return self._stack.pop() diff --git a/src/askui/telemetry/device_id.py b/src/askui/telemetry/device_id.py deleted file mode 100644 index 319f6d71..00000000 --- a/src/askui/telemetry/device_id.py +++ /dev/null @@ -1,24 +0,0 @@ -import logging - -import machineid - -logger = logging.getLogger(__name__) - -_HASH_KEY = "askui" - -_device_id: str | None = None - - -def get_device_id() -> str | None: - """Get the device ID (hashed) of the host device - - Returns None if the device ID is not found. - """ - global _device_id - if _device_id is None: - try: - _device_id = machineid.hashed_id(app_id=_HASH_KEY) - except machineid.MachineIdNotFound: - logger.debug("Device ID not found") - return None - return _device_id diff --git a/src/askui/telemetry/pkg_version.py b/src/askui/telemetry/pkg_version.py deleted file mode 100644 index 07c81b30..00000000 --- a/src/askui/telemetry/pkg_version.py +++ /dev/null @@ -1,44 +0,0 @@ -import logging -from importlib.metadata import Distribution, PackageNotFoundError, distribution, version - -logger = logging.getLogger(__name__) - - -def _get_module_name() -> str: - """Return the top-level package name from the module path.""" - if __name__ == "__main__": - error_msg = "This module is not meant to be run directly." - raise RuntimeError(error_msg) - - parts = __name__.split(".") - if not parts: - error_msg = "Failed to determine the module name - empty module path" - raise RuntimeError(error_msg) - - return parts[0] - - -def _get_distribution() -> Distribution | None: - """Get the distribution for the current package safely.""" - try: - module_name = _get_module_name() - return distribution(module_name) - except (PackageNotFoundError, RuntimeError) as e: - logger.warning("Failed to get distribution", extra={"error": str(e)}) - return None - - -def get_pkg_version() -> str: - """Return the package version or 'unknown' if version cannot be determined.""" - dist = _get_distribution() - if dist is None: - return "unknown" - - try: - return version(dist.name) - except PackageNotFoundError: - logger.debug( - 'Package not found. Setting version to "unknown".', - extra={"package_name": dist.name}, - ) - return "unknown" diff --git a/src/askui/telemetry/processors.py b/src/askui/telemetry/processors.py deleted file mode 100644 index 2cdc8870..00000000 --- a/src/askui/telemetry/processors.py +++ /dev/null @@ -1,123 +0,0 @@ -import abc -import logging -from datetime import datetime, timezone -from typing import Any, TypedDict - -import httpx -from pydantic import BaseModel, Field, HttpUrl - -from askui.telemetry.context import TelemetryContext - -logger = logging.getLogger(__name__) - - -class TelemetryProcessor(abc.ABC): - @abc.abstractmethod - def record_event( - self, - name: str, - attributes: dict[str, Any], - context: TelemetryContext, - ) -> None: ... - - @abc.abstractmethod - def flush(self) -> None: ... - - -class TelemetryEvent(TypedDict): - name: str - attributes: dict[str, Any] - context: TelemetryContext - timestamp: datetime - - -class SegmentSettings(BaseModel): - api_url: HttpUrl = Field( - default_factory=lambda: HttpUrl("https://tracking.askui.com") - ) - write_key: str = "Iae4oWbOo509Acu5ZeEb2ihqSpemjnhY" - timeout: int = 10 - max_retries: int = 3 - - -class Segment(TelemetryProcessor): - def __init__(self, settings: SegmentSettings) -> None: - self._settings = settings - - from segment import analytics # type: ignore - - self._analytics = analytics - self._analytics.write_key = settings.write_key - self._analytics.host = settings.api_url.encoded_string() - self._analytics.timeout = settings.timeout - self._analytics.max_retries = settings.max_retries - - def record_event( - self, - name: str, - attributes: dict[str, Any], - context: TelemetryContext, - ) -> None: - try: - self._analytics.track( - user_id=context.get("user_id"), - anonymous_id=context["anonymous_id"], - event=name, - properties={ - "attributes": attributes, - # Special context as Segment only supports predefined context keys - # (see https://segment.com/docs/connections/spec/track/#context) - "context": { - "os": context["os"], - "platform": context["platform"], - "session_id": context["session_id"], - "call_stack": context["call_stack"], - }, - }, - context={ - "app": context["app"], - "groupId": context.get("group_id"), - "os": { - "name": context["os"]["name"], - "version": context["os"]["version"], - }, - "device": context.get("device"), - }, - timestamp=datetime.now(tz=timezone.utc), - ) - except (ValueError, httpx.HTTPError) as e: - logger.debug( - "Failed to track event using Segment", - extra={"event_name": name, "error": str(e)}, - ) - - def flush(self) -> None: - self._analytics.shutdown() - - -class InMemoryProcessor(TelemetryProcessor): - def __init__(self) -> None: - self._events: list[TelemetryEvent] = [] - - def record_event( - self, - name: str, - attributes: dict[str, Any], - context: TelemetryContext, - ) -> None: - event: TelemetryEvent = { - "name": name, - "attributes": attributes, - "context": context, - "timestamp": datetime.now(tz=timezone.utc), - } - self._events.append(event) - - def get_events(self) -> list[TelemetryEvent]: - return self._events.copy() - - def clear(self) -> None: - self._events.clear() - - def flush(self) -> None: - pass diff --git a/src/askui/telemetry/telemetry.py b/src/askui/telemetry/telemetry.py deleted file mode 100644 index 6cc3cc07..00000000 --- a/src/askui/telemetry/telemetry.py +++ /dev/null @@ -1,285 +0,0 @@ -import inspect -import logging -import os -import platform -import time -import uuid -from collections.abc import Callable -from functools import wraps -from typing import Any - -from pydantic import BaseModel, Field -from typing_extensions import ParamSpec, TypeVar - -from askui.telemetry.anonymous_id import get_anonymous_id -from askui.telemetry.context import ( - AppContext, - CallStack, - DeviceContext, - OSContext, - PlatformContext, - TelemetryContext, -) -from askui.telemetry.pkg_version import get_pkg_version -from askui.telemetry.processors import SegmentSettings, TelemetryProcessor -from askui.telemetry.user_identification import ( - UserIdentification, - UserIdentificationSettings, -) - -logger = logging.getLogger(__name__) - -P = ParamSpec("P") -R = TypeVar("R") - - -class TelemetrySettings(BaseModel): - """Settings for telemetry configuration""" - - user_identification: UserIdentificationSettings | None = ( - UserIdentificationSettings() - ) - segment: SegmentSettings | None = SegmentSettings() - app_name: str = "askui-vision-agent" - app_version: str = get_pkg_version() - group_id: str | None = Field( - default=os.environ.get("ASKUI_WORKSPACE_ID"), - description=( - 'The group ID of the user. Defaults to the "ASKUI_WORKSPACE_ID" ' - "environment variable if set, otherwise `None`." - ), - ) - device_id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description=( - "The device ID of the host machine. " - "This is used to identify the device and the user (if anynomous) across " - "AskUI components. " - "We hash it with an AskUI specific salt to avoid user tracking across " - "(non-AskUI) applications or exposing the actual machine ID. This is the " - "trade-off we chose for now to protect user privacy while still being able " - "to improve the UX across components." - ), - ) - session_id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description=( - "This is used to identify the current (test/automation) session. " - "Defaults to a random UUID4. " - "This should be overridden if the test/automation is split across " - "multiple processes to associate all of them with a single session." - ), - ) - enabled: bool = True - - -class Telemetry: - _EXCLUDE_MASK = "masked" - - def __init__(self, settings: TelemetrySettings) -> None: - self._settings = settings - self._processors: list[TelemetryProcessor] = [] - self._user_identification: UserIdentification | None = None - if not self._settings.enabled: - logger.debug( - "Telemetry is disabled. To enable it, set the " - "`ASKUI__VA__TELEMETRY__ENABLED` environment variable to `True`." - ) - return - - logger.debug( - "Telemetry is enabled. To disable it, set the " - "`ASKUI__VA__TELEMETRY__ENABLED` environment variable to `False`." - ) - if self._settings.user_identification: - self._user_identification = UserIdentification( - settings=self._settings.user_identification - ) - self._call_stack = CallStack() - self._context = self._init_context() - - def set_processors(self, processors: list[TelemetryProcessor]) -> None: - """Set the telemetry processors that will be called in order - - *IMPORTANT*: This will replace the existing processors. - - Args: - processors (list[TelemetryProcessor]): The list of telemetry processors to set (may be empty) - """ - self._processors = processors - - def add_processor(self, processor: TelemetryProcessor) -> None: - """Add a telemetry processor that will be called in order of addition - - Args: - processor (TelemetryProcessor): The telemetry processor to add - """ - self._processors.append(processor) - - def _init_context(self) -> TelemetryContext: - context = TelemetryContext( - anonymous_id=get_anonymous_id(), - app=AppContext( - name=self._settings.app_name, - version=self._settings.app_version, - ), - os=OSContext( - name=platform.system(), - version=platform.version(), - release=platform.release(), - ), - platform=PlatformContext( - arch=platform.machine(), - python_version=platform.python_version(), - ), - session_id=self._settings.session_id, - ) - if self._settings.group_id: - context["group_id"] = self._settings.group_id - if self._user_identification: - user_id = self._user_identification.get_user_id() - if user_id: - context["user_id"] = user_id - if self._settings.device_id: - context["device"] = DeviceContext(id=self._settings.device_id) - return context - - def record_call( # noqa: C901 - self, - exclude: set[str] | None = None, - exclude_first_arg: bool = True, - exclude_response: bool = True, - exclude_exception: bool = False, - exclude_start: bool = True, - flush: bool = False, - ) -> Callable[[Callable[P, R]], Callable[P, R]]: - """Decorator to record calls to functions and methods - - IMPORTANT: Parameters, responses and exceptions recorded must be serializable - to JSON. Either make sure that they are serializable or exclude them using - the `exclude` parameter or use the `exclude_response` and `exclude_exception` - parameters. - - Args: - exclude (set[str] | None, optional): Set of parameters whose values are to - be excluded from tracking (masked to retain structure of the call). - Defaults to `None`. - exclude_first_arg (bool, optional): Whether to exclude the first argument, - e.g., `self` or `cls` for instance and class methods. Defaults to - `True`, for functions and static methods, it should be set to `False`. - exclude_response (bool, optional): Whether to exclude the response of the - function in the telemetry event. Defaults to `True`. - exclude_exception (bool, optional): Whether to exclude the exception if one - is raised in the telemetry event. Defaults to `False`. - exclude_start (bool, optional): Whether to exclude the start of the function - call as a telemetry event. Defaults to `True`. - flush (bool, optional): Whether to flush the telemetry data to the backend(s) - after recording an event. Defaults to `False`. Should be set to `True` if - an event. Defaults to `False`. Should be set to `True` if the process is - expected to exit afterwards. Setting it to `True` can have a slightly - negative impact on performance but ensures that telemetry data is not - lost in case of a crash. - """ - - _exclude = exclude or set() - - def decorator(func: Callable[P, R]) -> Callable[P, R]: # noqa: C901 - param_names_sorted = list(inspect.signature(func).parameters.keys()) - - @wraps(func) - def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: # noqa: C901 - if not self._settings.enabled: - return func(*args, **kwargs) - - self._call_stack.push_call() - module = func.__module__ - fn_name = func.__qualname__ - logger.debug( - "Record call", - extra={"fn_name": fn_name, "module_": module}, - ) - processed_args: tuple[Any, ...] = tuple( - arg if param_names_sorted[i] not in _exclude else self._EXCLUDE_MASK - for i, arg in enumerate(args) - ) - if exclude_first_arg: - processed_args = processed_args[1:] if processed_args else () - - def _serialize_arg(arg: Any) -> Any: - if to_telemetry := getattr(arg, "to_telemetry_dict", None): - if callable(to_telemetry): - return to_telemetry() - if isinstance(arg, BaseModel): - return arg.model_dump() - if inspect.isclass(arg): - return str(arg) - return arg - - processed_args = tuple(_serialize_arg(arg) for arg in processed_args) - processed_kwargs = { - k: v if k not in _exclude else self._EXCLUDE_MASK - for k, v in kwargs.items() - } - processed_kwargs = { - k: _serialize_arg(v) for k, v in processed_kwargs.items() - } - attributes: dict[str, Any] = { - "module": module, - "fn_name": fn_name, - "args": processed_args, - "kwargs": processed_kwargs, - "call_id": self._call_stack.current[-1], - } - self._context["call_stack"] = self._call_stack.current - if not exclude_start: - for processor in self._processors: - processor.record_event( - name="Function call started", - attributes=attributes, - context=self._context, - ) - start_time = time.time() - try: - response = func(*args, **kwargs) - duration_ms = (time.time() - start_time) * 1000 - attributes["duration_ms"] = duration_ms - if not exclude_response: - attributes["response"] = response - for processor in self._processors: - processor.record_event( - name="Function called", - attributes=attributes, - context=self._context, - ) - if flush: - self.flush() - except Exception as e: - duration_ms = (time.time() - start_time) * 1000 - attributes["duration_ms"] = duration_ms - if not exclude_exception: - attributes["exception"] = { - "type": type(e).__name__, - "message": str(e), - } - for processor in self._processors: - processor.record_event( - name="Function called", - attributes=attributes, - context=self._context, - ) - if flush: - self.flush() - raise - else: - return response - finally: - self._call_stack.pop_call() - - return wrapper - - return decorator - - def flush(self) -> None: - """Flush the telemetry data to the backend""" - for processor in self._processors: - processor.flush() diff --git a/src/askui/telemetry/user_identification.py b/src/askui/telemetry/user_identification.py deleted file mode 100644 index c421c52d..00000000 --- a/src/askui/telemetry/user_identification.py +++ /dev/null @@ -1,99 +0,0 @@ -import base64 -import logging -import os -import types -from functools import cached_property -from typing import Type, cast - -import httpx -from pydantic import BaseModel, Field, HttpUrl, SecretStr -from typing_extensions import Self - -logger = logging.getLogger(__name__) - - -def get_askui_token_from_env() -> SecretStr | None: - askui_token = os.environ.get("ASKUI_TOKEN") - if not askui_token: - return None - return SecretStr(askui_token) - - -class UserIdentificationSettings(BaseModel): - """Settings for user identification""" - - api_url: HttpUrl = HttpUrl("https://workspaces.askui.com/api/v1") - # retrieving directly through environment variable to circumvent pydantic-settings - # env_prefix - askui_token: SecretStr | None = Field(default=get_askui_token_from_env()) - askui_workspace_id: str | None = Field(default=os.environ.get("ASKUI_WORKSPACE_ID")) - verify_ssl: bool = Field( - default=(os.environ.get("ASKUI_HTTP_SSL_VERIFICATION", "True") == "True"), - description="Whether to use SSL verification for the AskUI Workspaces API.", - ) - - @cached_property - def askui_token_encoded(self) -> str | None: - if not self.askui_token: - return None - return base64.b64encode(self.askui_token.get_secret_value().encode()).decode() - - -class UserIdentification: - def __init__(self, settings: UserIdentificationSettings): - self._settings = settings - self._enabled = self._settings.askui_token and self._settings.askui_workspace_id - if not self._enabled: - logger.debug( - "User identification disabled. Set the `ASKUI_TOKEN` and " - "`ASKUI_WORKSPACE_ID` environment variables to allow user to be " - "identified." - ) - return - - self._client = httpx.Client( - timeout=30.0, - verify=self._settings.verify_ssl, - ) - - def __enter__(self) -> Self: - return self - - def __exit__( - self, - exc_type: Type[BaseException] | None, - exc_value: BaseException | None, - traceback: types.TracebackType | None, - ) -> None: - self._client.close() - - def get_user_id(self) -> str | None: - if not self._enabled: - return None - - try: - response = self._client.get( - f"{self._settings.api_url}/workspace-memberships", - params={ - "workspace_id": self._settings.askui_workspace_id, - "expand": "user", - }, - headers={ - "Authorization": f"Basic {self._settings.askui_token_encoded}", - "Content-Type": "application/json", - }, - ) - response.raise_for_status() - return cast( - "str | None", - response.json().get("data", [{}])[0].get("user", {}).get("id"), - ) - except httpx.HTTPError as e: - logger.debug("Failed to identify user", extra={"error": str(e)}) - except Exception as e: # noqa: BLE001 - We want to catch all other exceptions here - logger.debug( - "Unexpected error while identifying user", - extra={"error": str(e)}, - ) - - return None diff --git a/src/askui/telemetry/utils.py b/src/askui/telemetry/utils.py deleted file mode 100644 index 8c199fea..00000000 --- a/src/askui/telemetry/utils.py +++ /dev/null @@ -1,38 +0,0 @@ -import hashlib -import uuid - - -def is_valid_uuid4(uuid_str: str) -> bool: - """Check if string is a valid UUID4.""" - try: - return str(uuid.UUID(uuid_str, version=4)).lower() == uuid_str.lower() - except (ValueError, AttributeError): - return False - - -def hash_to_uuid4(value: str) -> str: - """Hash a string to a valid UUID4 string. - - The hashing is deterministic and collision resistant, meaning that it is highly - unlikely that two different inputs produce the same output. - - Args: - value: A string. - - Returns: - A string representation of a valid UUID4. - """ - b = value.encode() - digest = hashlib.sha256(b).digest() - - # Take the first 16 bytes to form the basis of our new UUID - raw_16 = bytearray(digest[:16]) - - # Force the UUID to be version 4: - # - The high nibble of byte 6 should be 0x4 - raw_16[6] = (raw_16[6] & 0x0F) | 0x40 - - # Force the UUID to be variant 1 (i.e., 10xx in binary): - raw_16[8] = (raw_16[8] & 0x3F) | 0x80 - new_uuid = uuid.UUID(bytes=bytes(raw_16)) - return str(new_uuid) diff --git a/src/askui/tools/askui/askui_controller.py b/src/askui/tools/askui/askui_controller.py index e0e0f1ad..0f1761f8 100644 --- a/src/askui/tools/askui/askui_controller.py +++ b/src/askui/tools/askui/askui_controller.py @@ -12,7 +12,6 @@ from PIL import Image from typing_extensions import Self, override -from askui.container import telemetry from askui.reporting import NULL_REPORTER, Reporter from askui.tools.agent_os import ( ComputerAgentOS, @@ -119,7 +118,6 @@ class MultiComputerTargetAgentOS(ComputerAgentOS): _REPORTER_SOURCE = "AgentOS" - @telemetry.record_call(exclude={"reporter", "agent_os_target_computers"}) def __init__( self, reporter: Reporter = NULL_REPORTER, @@ -154,7 +152,6 @@ def _require_active_agent_os_target_computer(self) -> ComputerTarget: def _session_info(self) -> controller_v1_pbs.SessionInfo: return self._manager.active_connection().session_info - @telemetry.record_call(exclude={"agent_os_target_computer"}) @override def add_agent_os_target_computer( self, agent_os_target_computer: ComputerTarget @@ -170,7 +167,6 @@ def add_agent_os_target_computer( self._manager.add(agent_os_target_computer) return agent_os_target_computer - @telemetry.record_call(exclude={"agent_os_target_computers"}) @override def reset_agent_os_target_computers( self, @@ -201,7 +197,6 @@ def reset_agent_os_target_computers( if was_connected: self.connect() - @telemetry.record_call() @override def describe_agent_os_target_computers(self) -> list[str]: """Return the `repr()` string of every registered target computer.""" @@ -216,7 +211,6 @@ def describe_agent_os_target_computers(self) -> list[str]: ) return agent_os_target_computer_reprs - @telemetry.record_call() @override def get_current_computer_target_id(self, report: bool = True) -> str: """Return the `computer_id` of the currently active Agent OS target computer.""" @@ -232,7 +226,6 @@ def get_current_computer_target_id(self, report: bool = True) -> str: ) return computer_id - @telemetry.record_call() @override def switch_agent_os_target_computer(self, computer_id: str) -> ComputerTarget: """ @@ -282,7 +275,6 @@ def temporary_select(self, computer_id: str) -> Iterator[Self]: f"temporary_select({computer_id!r}) -> restored", ) - @telemetry.record_call() @override def connect(self) -> None: """ @@ -342,7 +334,6 @@ def _run_recorder_action( ) return response - @telemetry.record_call() @override def disconnect(self) -> None: """ @@ -351,7 +342,6 @@ def disconnect(self) -> None: """ self._manager.disconnect() - @telemetry.record_call() def __enter__(self) -> Self: """ Context manager entry point that establishes the connection. @@ -362,7 +352,6 @@ def __enter__(self) -> Self: self.connect() return self - @telemetry.record_call(exclude={"exc_value", "traceback"}) def __exit__( self, exc_type: Type[BaseException] | None, @@ -401,7 +390,6 @@ def _check_bitmap_dimensions(width: int, height: int) -> None: ) raise AskUiControllerError(error_msg) - @telemetry.record_call() @override def screenshot(self, report: bool = True, unscaled: bool = False) -> Image.Image: """ @@ -439,7 +427,6 @@ def screenshot(self, report: bool = True, unscaled: bool = False) -> Image.Image self._reporter.add_message(self._REPORTER_SOURCE, "screenshot()", image) return image - @telemetry.record_call() @override def mouse_move(self, x: int, y: int, duration: int = 500) -> None: """ @@ -465,7 +452,6 @@ def mouse_move(self, x: int, y: int, duration: int = 500) -> None: ), ) - @telemetry.record_call(exclude={"text"}) @override def type(self, text: str, typing_speed: int = 50) -> None: """ @@ -490,7 +476,6 @@ def type(self, text: str, typing_speed: int = 50) -> None: ), ) - @telemetry.record_call() @override def click( self, button: Literal["left", "middle", "right"] = "left", count: int = 1 @@ -521,7 +506,6 @@ def click( ), ) - @telemetry.record_call() @override def mouse_down(self, button: Literal["left", "middle", "right"] = "left") -> None: """ @@ -549,7 +533,6 @@ def mouse_down(self, button: Literal["left", "middle", "right"] = "left") -> Non ), ) - @telemetry.record_call() @override def mouse_up(self, button: Literal["left", "middle", "right"] = "left") -> None: """ @@ -577,7 +560,6 @@ def mouse_up(self, button: Literal["left", "middle", "right"] = "left") -> None: ), ) - @telemetry.record_call() @override def mouse_scroll(self, dx: int, dy: int) -> None: """ @@ -615,7 +597,6 @@ def mouse_scroll(self, dx: int, dy: int) -> None: ), ) - @telemetry.record_call() @override def keyboard_pressed( self, key: PcKey | ModifierKey, modifier_keys: list[ModifierKey] | None = None @@ -642,7 +623,6 @@ def keyboard_pressed( ), ) - @telemetry.record_call() @override def keyboard_release( self, key: PcKey | ModifierKey, modifier_keys: list[ModifierKey] | None = None @@ -669,7 +649,6 @@ def keyboard_release( ), ) - @telemetry.record_call() @override def keyboard_tap( self, @@ -702,7 +681,6 @@ def keyboard_tap( ), ) - @telemetry.record_call() @override def set_display(self, display: int = 1) -> None: """ @@ -719,7 +697,6 @@ def set_display(self, display: int = 1) -> None: self._require_active_agent_os_target_computer().display = display self._reporter.add_message(self._REPORTER_SOURCE, f"set_display({display})") - @telemetry.record_call(exclude={"command"}) @override def run_command(self, command: str, timeout_ms: int = 30000) -> None: """ @@ -742,7 +719,6 @@ def run_command(self, command: str, timeout_ms: int = 30000) -> None: ), ) - @telemetry.record_call() @override def retrieve_active_display(self) -> Display: """ @@ -773,7 +749,6 @@ def retrieve_active_display(self) -> Display: ) raise ValueError(error_msg) - @telemetry.record_call() @override def list_displays( self, @@ -806,7 +781,6 @@ def list_displays( return displays - @telemetry.record_call() def get_process_list( self, get_extended_info: bool = False ) -> controller_v1_pbs.Response_GetProcessList: @@ -841,7 +815,6 @@ def get_process_list( return response - @telemetry.record_call() def get_window_list( self, process_id: int ) -> controller_v1_pbs.Response_GetWindowList: @@ -872,7 +845,6 @@ def get_window_list( return response - @telemetry.record_call() def get_automation_target_list( self, ) -> controller_v1_pbs.Response_GetAutomationTargetList: @@ -898,7 +870,6 @@ def get_automation_target_list( return response - @telemetry.record_call() def set_mouse_delay(self, delay_ms: int) -> None: """ Configure mouse action delay. @@ -917,7 +888,6 @@ def set_mouse_delay(self, delay_ms: int) -> None: ) ) - @telemetry.record_call() def set_keyboard_delay(self, delay_ms: int) -> None: """ Configure keyboard action delay. @@ -936,7 +906,6 @@ def set_keyboard_delay(self, delay_ms: int) -> None: ) ) - @telemetry.record_call() def set_active_window(self, process_id: int, window_id: int) -> int: """ Set the active window for automation. @@ -981,7 +950,6 @@ def set_active_window(self, process_id: int, window_id: int) -> int: ) return new_display_length - @telemetry.record_call() def set_active_automation_target(self, target_id: int) -> None: """ Set the active automation target. @@ -998,7 +966,6 @@ def set_active_automation_target(self, target_id: int) -> None: controller_v1_pbs.Request_SetActiveAutomationTarget(ID=target_id) ) - @telemetry.record_call() def schedule_batched_action( self, action_class_id: controller_v1_pbs.ActionClassID, @@ -1035,7 +1002,6 @@ def schedule_batched_action( return response - @telemetry.record_call() def start_batch_run(self) -> None: """ Start executing batched actions. @@ -1047,7 +1013,6 @@ def start_batch_run(self) -> None: controller_v1_pbs.Request_StartBatchRun(sessionInfo=self._session_info) ) - @telemetry.record_call() def stop_batch_run(self) -> None: """ Stop executing batched actions. @@ -1059,7 +1024,6 @@ def stop_batch_run(self) -> None: controller_v1_pbs.Request_StopBatchRun(sessionInfo=self._session_info) ) - @telemetry.record_call() def get_action_count(self) -> controller_v1_pbs.Response_GetActionCount: """ Get the count of recorded or batched actions. @@ -1079,7 +1043,6 @@ def get_action_count(self) -> controller_v1_pbs.Response_GetActionCount: ) return response - @telemetry.record_call() def get_action(self, action_index: int) -> controller_v1_pbs.Response_GetAction: """ Get a specific action by its index. @@ -1104,7 +1067,6 @@ def get_action(self, action_index: int) -> controller_v1_pbs.Response_GetAction: return response - @telemetry.record_call() def remove_action(self, action_id: int) -> None: """ Remove a specific action by its ID. @@ -1121,7 +1083,6 @@ def remove_action(self, action_id: int) -> None: ) ) - @telemetry.record_call() def remove_all_actions(self) -> None: """ Clear all recorded or batched actions. @@ -1169,7 +1130,6 @@ def _send_command(self, command: Command) -> AskUIAgentOSSendResponseSchema: return AskUIAgentOSSendResponseSchema.model_validate_json(response.message) - @telemetry.record_call() def get_mouse_position(self) -> Coordinate: """ Get the mouse cursor position @@ -1188,7 +1148,6 @@ def get_mouse_position(self) -> Coordinate: ) return coordinate - @telemetry.record_call() def set_mouse_position(self, x: int, y: int) -> None: """ Set the mouse cursor position to specific coordinates. @@ -1204,7 +1163,6 @@ def set_mouse_position(self, x: int, y: int) -> None: ) self._send_command(command) - @telemetry.record_call() def render_quad(self, style: RenderObjectStyle) -> int: """ Render a quad object to the display. @@ -1220,7 +1178,6 @@ def render_quad(self, style: RenderObjectStyle) -> int: res = self._send_command(command) return int(res.message.command.response.id.root) # type: ignore[union-attr] - @telemetry.record_call() def render_line(self, style: RenderObjectStyle, points: list[Coordinate]) -> int: """ Render a line object to the display. @@ -1239,7 +1196,6 @@ def render_line(self, style: RenderObjectStyle, points: list[Coordinate]) -> int res = self._send_command(command) return int(res.message.command.response.id.root) # type: ignore[union-attr] - @telemetry.record_call(exclude={"image_data"}) def render_image(self, style: RenderObjectStyle, image_data: str) -> int: """ Render an image object to the display. @@ -1260,7 +1216,6 @@ def render_image(self, style: RenderObjectStyle, image_data: str) -> int: return int(res.message.command.response.id.root) # type: ignore[union-attr] - @telemetry.record_call() def render_text(self, style: RenderObjectStyle, content: str) -> int: """ Render a text object to the display. @@ -1280,7 +1235,6 @@ def render_text(self, style: RenderObjectStyle, content: str) -> int: res = self._send_command(command) return int(res.message.command.response.id.root) # type: ignore[union-attr] - @telemetry.record_call() def update_render_object(self, object_id: int, style: RenderObjectStyle) -> None: """ Update styling properties of an existing render object. @@ -1299,7 +1253,6 @@ def update_render_object(self, object_id: int, style: RenderObjectStyle) -> None command = UpdateRenderObjectCommand(parameters=[render_object_id, style]) self._send_command(command) - @telemetry.record_call() def delete_render_object(self, object_id: int) -> None: """ Delete an existing render object from the display. @@ -1314,7 +1267,6 @@ def delete_render_object(self, object_id: int) -> None: command = DeleteRenderObjectCommand(parameters=[render_object_id]) self._send_command(command) - @telemetry.record_call() def clear_render_objects(self) -> None: """ Clear all render objects from the display. @@ -1420,7 +1372,6 @@ def set_window_in_focus(self, process_id: int, window_id: int) -> None: command = SetActiveWindowCommand(parameters=[_process_id, _window_id]) self._send_command(command) - @telemetry.record_call() @override def get_file_names(self, absolute_directory_path: str) -> list[str]: """ @@ -1453,7 +1404,6 @@ def get_file_names(self, absolute_directory_path: str) -> list[str]: ) return res.response.fileNames - @telemetry.record_call() @override def get_file(self, path: str) -> Image.Image | PdfSource | str: """ @@ -1510,7 +1460,6 @@ def get_file(self, path: str) -> Image.Image | PdfSource | str: ) return decoded - @telemetry.record_call() @override def remove_virtual_displays(self) -> None: """ diff --git a/src/askui/web_agent.py b/src/askui/web_agent.py index 4df9c97a..faa8fdca 100644 --- a/src/askui/web_agent.py +++ b/src/askui/web_agent.py @@ -6,7 +6,6 @@ from askui.agent_base import Agent from askui.agent_settings import AgentSettings from askui.callbacks import ConversationCallback -from askui.container import telemetry from askui.models.shared.secrets import Secret from askui.models.shared.settings import ( ActSettings, @@ -88,18 +87,6 @@ class WebAgent(Agent): ``` """ - @telemetry.record_call( - exclude={ - "reporters", - "settings", - "act_tools", - "callbacks", - "truncation_strategy", - "secrets", - "download_dir", - "auto_follow_new_tab", - } - ) @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def __init__( self, diff --git a/tests/conftest.py b/tests/conftest.py index f2a792af..c8f968be 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -107,13 +107,6 @@ def agent_os_mock_patch( return agent_os_mock -@pytest.fixture(autouse=True) -def disable_telemetry() -> None: - from askui.container import telemetry - - telemetry.set_processors([]) - - @pytest.fixture def askui_logo_bmp(path_fixtures: pathlib.Path) -> Image.Image: """Fixture providing askui logo as BMP.""" diff --git a/tests/e2e/test_telemetry.py b/tests/e2e/test_telemetry.py deleted file mode 100644 index 70539277..00000000 --- a/tests/e2e/test_telemetry.py +++ /dev/null @@ -1,28 +0,0 @@ -import pytest -from PIL import Image - -from askui import ComputerAgent -from askui import locators as loc -from askui.container import telemetry -from askui.telemetry.processors import Segment, SegmentSettings -from askui.tools.agent_os import ComputerAgentOS - - -@pytest.mark.timeout(60) -def test_telemetry_with_nonexistent_domain_should_not_block( - github_login_screenshot: Image.Image, - agent_os_mock_patch: ComputerAgentOS, # noqa: ARG001 -) -> None: - telemetry.set_processors( - [ - Segment( - SegmentSettings( - api_url="https://this-domain-does-not-exist-123456789.com", - write_key="1234567890", - ) - ) - ] - ) - with ComputerAgent() as agent: - agent.locate(loc.Text(), screenshot=github_login_screenshot) - assert True diff --git a/tests/unit/telemetry/__init__.py b/tests/unit/telemetry/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/unit/telemetry/test_anonymous_id.py b/tests/unit/telemetry/test_anonymous_id.py deleted file mode 100644 index 2580b2a1..00000000 --- a/tests/unit/telemetry/test_anonymous_id.py +++ /dev/null @@ -1,105 +0,0 @@ -import uuid -from collections.abc import Generator - -import machineid -import pytest -from pytest_mock import MockerFixture - -from askui.telemetry.anonymous_id import get_anonymous_id - - -@pytest.fixture(autouse=True) -def reset_caches() -> Generator[None, None, None]: - """Reset the module-level caches before each test.""" - import askui.telemetry.anonymous_id - import askui.telemetry.device_id - - askui.telemetry.anonymous_id._anonymous_id = None - askui.telemetry.device_id._device_id = None - yield - askui.telemetry.anonymous_id._anonymous_id = None - askui.telemetry.device_id._device_id = None - - -def test_get_anonymous_id_returns_cached_id() -> None: - # First call to get_anonymous_id will set the cache - first_id = get_anonymous_id() - - # Second call should return the same ID - second_id = get_anonymous_id() - assert first_id == second_id - - -def test_get_anonymous_id_uses_device_id_when_available(mocker: MockerFixture) -> None: - test_device_id = "test-device-id" - expected_anonymous_id = "7c810ac9-d1be-4620-a665-95f9554920ec" - - mocker.patch( - "askui.telemetry.anonymous_id._read_anonymous_id_from_file", return_value=None - ) - mocker.patch("askui.telemetry.device_id.get_device_id", return_value=test_device_id) - mocker.patch("askui.telemetry.utils.is_valid_uuid4", return_value=False) - mocker.patch( - "askui.telemetry.utils.hash_to_uuid4", return_value=expected_anonymous_id - ) - mocker.patch( - "askui.telemetry.anonymous_id.hash_to_uuid4", return_value=expected_anonymous_id - ) - mock_write = mocker.patch( - "askui.telemetry.anonymous_id._write_anonymous_id_to_file" - ) - - anonymous_id = get_anonymous_id() - assert anonymous_id == expected_anonymous_id - mock_write.assert_called_once_with(expected_anonymous_id) - - -def test_get_anonymous_id_uses_random_uuid_when_device_id_unavailable( - mocker: MockerFixture, -) -> None: - test_id = "1c9cb557-1c83-45b0-8aa8-6938c84d5893" - - mocker.patch( - "askui.telemetry.anonymous_id._read_anonymous_id_from_file", return_value=None - ) - mocker.patch("machineid.hashed_id", side_effect=machineid.MachineIdNotFound()) - mocker.patch("askui.telemetry.device_id.get_device_id", return_value=None) - mocker.patch("askui.telemetry.utils.is_valid_uuid4", return_value=False) - mocker.patch("uuid.uuid4", return_value=uuid.UUID(test_id.upper())) - mock_write = mocker.patch( - "askui.telemetry.anonymous_id._write_anonymous_id_to_file" - ) - - anonymous_id = get_anonymous_id() - assert anonymous_id == test_id - mock_write.assert_called_once_with(test_id) - - -def test_get_anonymous_id_reads_from_file(mocker: MockerFixture) -> None: - test_id = "1c9cb557-1c83-45b0-8aa8-6938c84d5893" - mocker.patch( - "askui.telemetry.anonymous_id._read_anonymous_id_from_file", - return_value=test_id, - ) - mocker.patch("askui.telemetry.utils.is_valid_uuid4", return_value=True) - - anonymous_id = get_anonymous_id() - assert anonymous_id == test_id - - -def test_get_anonymous_id_writes_to_file(mocker: MockerFixture) -> None: - test_id = "1c9cb557-1c83-45b0-8aa8-6938c84d5893" - - mocker.patch( - "askui.telemetry.anonymous_id._read_anonymous_id_from_file", return_value=None - ) - mocker.patch("machineid.hashed_id", side_effect=machineid.MachineIdNotFound()) - mocker.patch("askui.telemetry.device_id.get_device_id", return_value=None) - mocker.patch("askui.telemetry.utils.is_valid_uuid4", return_value=False) - mocker.patch("uuid.uuid4", return_value=uuid.UUID(test_id.upper())) - mock_write = mocker.patch( - "askui.telemetry.anonymous_id._write_anonymous_id_to_file" - ) - - get_anonymous_id() - mock_write.assert_called_once_with(test_id) diff --git a/tests/unit/telemetry/test_device_id.py b/tests/unit/telemetry/test_device_id.py deleted file mode 100644 index 620a575c..00000000 --- a/tests/unit/telemetry/test_device_id.py +++ /dev/null @@ -1,33 +0,0 @@ -import machineid -from pytest_mock import MockerFixture - -from askui.telemetry.device_id import get_device_id - - -def test_get_device_id_returns_cached_id(mocker: MockerFixture) -> None: - # First call to get_device_id will set the cache - mocker.patch("askui.telemetry.device_id._device_id", None) - mocker.patch( - "machineid.hashed_id", - return_value="02c2431a4608f230d2d759ac888d773d274229ebd9c9093249752dd839ee3ea3", - ) - first_id = get_device_id() - - # Second call should return the same ID - second_id = get_device_id() - assert first_id == second_id - - -def test_get_device_id_returns_hashed_id(mocker: MockerFixture) -> None: - test_id = "02c2431a4608f230d2d759ac888d773d274229ebd9c9093249752dd839ee3ea3" - mocker.patch("askui.telemetry.device_id._device_id", None) - mocker.patch("machineid.hashed_id", return_value=test_id) - device_id = get_device_id() - assert device_id == test_id - - -def test_get_device_id_returns_none_on_error(mocker: MockerFixture) -> None: - mocker.patch("askui.telemetry.device_id._device_id", None) - mocker.patch("machineid.hashed_id", side_effect=machineid.MachineIdNotFound) - device_id = get_device_id() - assert device_id is None diff --git a/tests/unit/telemetry/test_telemetry.py b/tests/unit/telemetry/test_telemetry.py deleted file mode 100644 index 2e55b125..00000000 --- a/tests/unit/telemetry/test_telemetry.py +++ /dev/null @@ -1,584 +0,0 @@ -from typing import Any - -import pytest - -from askui.telemetry import InMemoryProcessor, Telemetry, TelemetrySettings - - -def test_telemetry_disabled() -> None: - settings = TelemetrySettings(enabled=False) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - @telemetry.record_call() - def test_func(x: int) -> int: - return x * 2 - - result = test_func(5) - assert result == 10 - assert len(processor.get_events()) == 0 - - -def test_telemetry_enabled() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - @telemetry.record_call( - exclude_first_arg=False, exclude_response=False, exclude_start=False - ) - def test_func(x: int) -> int: - return x * 2 - - result = test_func(5) - assert result == 10 - - events = processor.get_events() - assert len(events) == 2 - - start_event = events[0] - assert start_event["name"] == "Function call started" - assert start_event["attributes"]["fn_name"].endswith("test_func") - assert start_event["attributes"]["args"] == (5,) - assert start_event["attributes"]["kwargs"] == {} - - end_event = events[1] - assert end_event["name"] == "Function called" - assert end_event["attributes"]["fn_name"].endswith("test_func") - assert end_event["attributes"]["args"] == (5,) - assert end_event["attributes"]["kwargs"] == {} - assert end_event["attributes"]["response"] == 10 - assert end_event["attributes"]["duration_ms"] is not None - assert end_event["attributes"]["duration_ms"] >= 0 - - -def test_telemetry_error() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - @telemetry.record_call( - exclude_first_arg=False, exclude_start=False, exclude_exception=False - ) - def test_func(x: int) -> int: # noqa: ARG001 - error_msg = "Test error" - raise ValueError(error_msg) - - with pytest.raises(ValueError): - test_func(5) - - events = processor.get_events() - assert len(events) == 2 - - start_event = events[0] - assert start_event["name"] == "Function call started" - assert start_event["attributes"]["fn_name"].endswith("test_func") - assert start_event["attributes"]["args"] == (5,) - assert start_event["attributes"]["kwargs"] == {} - - error_event = events[1] - assert error_event["name"] == "Function called" - assert error_event["attributes"]["fn_name"].endswith("test_func") - assert error_event["attributes"]["args"] == (5,) - assert error_event["attributes"]["kwargs"] == {} - assert error_event["attributes"]["exception"]["type"] == "ValueError" - assert error_event["attributes"]["exception"]["message"] == "Test error" - assert error_event["attributes"]["duration_ms"] is not None - assert error_event["attributes"]["duration_ms"] >= 0 - - -def test_multiple_processors() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor1 = InMemoryProcessor() - processor2 = InMemoryProcessor() - telemetry.add_processor(processor1) - telemetry.add_processor(processor2) - - @telemetry.record_call(exclude_start=False) - def test_func(x: int) -> int: - return x * 2 - - result = test_func(5) - assert result == 10 - - events1 = processor1.get_events() - events2 = processor2.get_events() - assert len(events1) == 2 - assert len(events2) == 2 - for e1, e2 in zip(events1, events2, strict=False): - assert e1["name"] == e2["name"] - assert e1["attributes"]["fn_name"] == e2["attributes"]["fn_name"] - assert e1["attributes"]["args"] == e2["attributes"]["args"] - assert e1["attributes"]["kwargs"] == e2["attributes"]["kwargs"] - if "response" in e1["attributes"]: - assert e1["attributes"]["response"] == e2["attributes"]["response"] - if "exception" in e1["attributes"]: - assert e1["attributes"]["exception"] == e2["attributes"]["exception"] - assert e1["attributes"]["duration_ms"] == e2["attributes"]["duration_ms"] - assert e1["timestamp"] <= e2["timestamp"] - - -def test_function_tracking() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - @telemetry.record_call(exclude_start=False) - def standalone_function(x: int) -> int: - return x * 2 - - result = standalone_function(5) - assert result == 10 - - events = processor.get_events() - assert len(events) == 2 - assert events[0]["attributes"]["fn_name"].endswith("standalone_function") - assert events[1]["attributes"]["fn_name"].endswith("standalone_function") - - -def test_instance_method_tracking() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - class TestClass: - @telemetry.record_call(exclude_start=False) - def instance_method(self, x: int) -> int: - return x * 2 - - obj = TestClass() - result = obj.instance_method(5) - assert result == 10 - - events = processor.get_events() - assert len(events) == 2 - assert events[0]["attributes"]["fn_name"].endswith("TestClass.instance_method") - assert events[1]["attributes"]["fn_name"].endswith("TestClass.instance_method") - - -def test_class_method_tracking() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - class TestClass: - @classmethod - @telemetry.record_call(exclude_start=False) - def class_method(cls, x: int) -> int: - return x * 3 - - result = TestClass.class_method(5) - assert result == 15 - - events = processor.get_events() - assert len(events) == 2 - assert events[0]["attributes"]["fn_name"].endswith("TestClass.class_method") - assert events[1]["attributes"]["fn_name"].endswith("TestClass.class_method") - - -def test_static_method_tracking() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - class TestClass: - @staticmethod - @telemetry.record_call(exclude_start=False) - def static_method(x: int) -> int: - return x * 4 - - result = TestClass.static_method(5) - assert result == 20 - - events = processor.get_events() - assert len(events) == 2 - assert events[0]["attributes"]["fn_name"].endswith("TestClass.static_method") - assert events[1]["attributes"]["fn_name"].endswith("TestClass.static_method") - - -def test_nested_class_tracking() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - class Outer: - class Inner: - @telemetry.record_call(exclude_start=False) - def nested_method(self, x: int) -> int: - return x * 2 - - result = Outer.Inner().nested_method(5) - assert result == 10 - - events = processor.get_events() - assert len(events) == 2 - assert events[0]["attributes"]["fn_name"].endswith("Outer.Inner.nested_method") - assert events[1]["attributes"]["fn_name"].endswith("Outer.Inner.nested_method") - - -def test_exclude_parameter() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - @telemetry.record_call( - exclude={"password", "token"}, exclude_first_arg=False, exclude_start=False - ) - def sensitive_function(username: str, password: str, token: str) -> str: # noqa: ARG001 - return f"User: {username}" - - result = sensitive_function("test_user", "secret_password", "private_token") - assert result == "User: test_user" - - events = processor.get_events() - assert len(events) == 2 - - # Check that excluded parameters are masked - start_event = events[0] - assert start_event["attributes"]["args"][0] == "test_user" # username is included - assert start_event["attributes"]["args"][1] == "masked" # password is masked - assert start_event["attributes"]["args"][2] == "masked" # token is masked - - end_event = events[1] - assert end_event["attributes"]["args"][0] == "test_user" - assert end_event["attributes"]["args"][1] == "masked" - assert end_event["attributes"]["args"][2] == "masked" - - -def test_exclude_kwargs() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - @telemetry.record_call( - exclude={"password", "token"}, exclude_first_arg=False, exclude_start=False - ) - def sensitive_function(username: str, **kwargs: Any) -> str: # noqa: ARG001, ARG002 - return f"User: {username}" - - result = sensitive_function( - "test_user", password="secret_password", token="private_token", visible="ok" - ) - assert result == "User: test_user" - - events = processor.get_events() - assert len(events) == 2 - - # Check that excluded kwargs are masked but others aren't - start_event = events[0] - assert start_event["attributes"]["args"][0] == "test_user" - assert start_event["attributes"]["kwargs"]["password"] == "masked" - assert start_event["attributes"]["kwargs"]["token"] == "masked" - assert start_event["attributes"]["kwargs"]["visible"] == "ok" - - -def test_include_first_arg_function() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - @telemetry.record_call(exclude_first_arg=False, exclude_start=False) - def test_func(first: str, second: str) -> str: - return f"{first}-{second}" - - result = test_func("one", "two") - assert result == "one-two" - - events = processor.get_events() - assert len(events) == 2 - - # Check that first argument is included - assert events[0]["attributes"]["args"] == ("one", "two") - assert events[1]["attributes"]["args"] == ("one", "two") - - -def test_include_first_arg_method() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - class TestClass: - def __init__(self, name: str): - self.name = name - - @telemetry.record_call(exclude_first_arg=False, exclude_start=False) - def method_with_self(self, param: str) -> str: - return f"{self.name}-{param}" - - obj = TestClass("test") - result = obj.method_with_self("param") - assert result == "test-param" - - events = processor.get_events() - assert len(events) == 2 - - # Check that self is included as first argument - assert len(events[0]["attributes"]["args"]) == 2 - assert events[0]["attributes"]["args"][1] == "param" # Second arg should be param - # Can't directly check self, but we can check it's not removed - - -def test_default_exclude_self_method() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - class TestClass: - def __init__(self, name: str): - self.name = name - - @telemetry.record_call(exclude_start=False) # Default exclude_first_arg=True - def method_with_self(self, param: str) -> str: - return f"{self.name}-{param}" - - obj = TestClass("test") - result = obj.method_with_self("param") - assert result == "test-param" - - events = processor.get_events() - assert len(events) == 2 - - # Check that self is excluded - assert events[0]["attributes"]["args"] == ("param",) - assert events[1]["attributes"]["args"] == ("param",) - - -def test_combined_exclude_and_include_first_arg() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - class User: - @telemetry.record_call( - exclude={"password"}, exclude_first_arg=False, exclude_start=False - ) - def authenticate(self, username: str, password: str) -> bool: - return username == "valid" and password == "correct" - - user = User() - result = user.authenticate("valid", "correct") - assert result is True - - events = processor.get_events() - assert len(events) == 2 - - # First arg (self) should be included, password should be masked - assert len(events[0]["attributes"]["args"]) == 3 - assert events[0]["attributes"]["args"][1] == "valid" # username is included - assert events[0]["attributes"]["args"][2] == "masked" # password is masked - - assert len(events[1]["attributes"]["args"]) == 3 - assert events[1]["attributes"]["args"][1] == "valid" - assert events[1]["attributes"]["args"][2] == "masked" - - -def test_static_method_with_include_first_arg() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - class TestClass: - @staticmethod - @telemetry.record_call(exclude_first_arg=False, exclude_start=False) - def static_method(first: str, second: str) -> str: - return f"{first}-{second}" - - result = TestClass.static_method("one", "two") - assert result == "one-two" - - events = processor.get_events() - assert len(events) == 2 - - # Check that all arguments are included - assert events[0]["attributes"]["args"] == ("one", "two") - assert events[1]["attributes"]["args"] == ("one", "two") - - -# Tests for new parameters -def test_exclude_response() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - @telemetry.record_call(exclude_response=True, exclude_start=False) - def test_func(x: int) -> int: - return x * 2 - - result = test_func(5) - assert result == 10 - - events = processor.get_events() - assert len(events) == 2 - - # Check that response is excluded - end_event = events[1] - assert "response" not in end_event["attributes"] - - -def test_include_response() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - @telemetry.record_call(exclude_response=False, exclude_start=False) - def test_func(x: int) -> int: - return x * 2 - - result = test_func(5) - assert result == 10 - - events = processor.get_events() - assert len(events) == 2 - - # Check that response is included - end_event = events[1] - assert "response" in end_event["attributes"] - assert end_event["attributes"]["response"] == 10 - - -def test_exclude_exception() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - @telemetry.record_call(exclude_exception=True, exclude_start=False) - def test_func(x: int) -> int: # noqa: ARG001 - error_msg = "Test error" - raise ValueError(error_msg) - - with pytest.raises(ValueError): - test_func(5) - - events = processor.get_events() - assert len(events) == 2 - - # Check that exception is excluded - end_event = events[1] - assert "exception" not in end_event["attributes"] - - -def test_include_exception() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - @telemetry.record_call(exclude_exception=False, exclude_start=False) - def test_func(x: int) -> int: # noqa: ARG001 - error_msg = "Test error" - raise ValueError(error_msg) - - with pytest.raises(ValueError): - test_func(5) - - events = processor.get_events() - assert len(events) == 2 - - # Check that exception is included - end_event = events[1] - assert "exception" in end_event["attributes"] - assert end_event["attributes"]["exception"]["type"] == "ValueError" - assert end_event["attributes"]["exception"]["message"] == "Test error" - - -def test_exclude_start() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - @telemetry.record_call(exclude_start=True) - def test_func(x: int) -> int: - return x * 2 - - result = test_func(5) - assert result == 10 - - events = processor.get_events() - assert len(events) == 1 - - # Check that only the end event is recorded - end_event = events[0] - assert end_event["name"] == "Function called" - - -def test_include_start() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - @telemetry.record_call(exclude_start=False) - def test_func(x: int) -> int: - return x * 2 - - result = test_func(5) - assert result == 10 - - events = processor.get_events() - assert len(events) == 2 - - # Check that both start and end events are recorded - start_event = events[0] - end_event = events[1] - assert start_event["name"] == "Function call started" - assert end_event["name"] == "Function called" - - -def test_all_new_parameters_together() -> None: - settings = TelemetrySettings(enabled=True) - telemetry = Telemetry(settings) - processor = InMemoryProcessor() - telemetry.add_processor(processor) - - @telemetry.record_call( - exclude_response=False, exclude_exception=False, exclude_start=False - ) - def test_func(x: int) -> int: - if x < 0: - error_msg = "Negative input" - raise ValueError(error_msg) - return x * 2 - - # Successful case - result = test_func(5) - assert result == 10 - - # Error case - with pytest.raises(ValueError): - test_func(-5) - - events = processor.get_events() - assert len(events) == 4 - - # First pair of events (successful call) - assert events[0]["name"] == "Function call started" - assert events[1]["name"] == "Function called" - assert "response" in events[1]["attributes"] - assert events[1]["attributes"]["response"] == 10 - - # Second pair of events (error call) - assert events[2]["name"] == "Function call started" - assert events[3]["name"] == "Function called" - assert "exception" in events[3]["attributes"] - assert events[3]["attributes"]["exception"]["type"] == "ValueError" - assert events[3]["attributes"]["exception"]["message"] == "Negative input" diff --git a/tests/unit/telemetry/test_utils.py b/tests/unit/telemetry/test_utils.py deleted file mode 100644 index 23b6ba0c..00000000 --- a/tests/unit/telemetry/test_utils.py +++ /dev/null @@ -1,65 +0,0 @@ -import uuid - -from askui.telemetry.utils import hash_to_uuid4, is_valid_uuid4 - - -def test_is_valid_uuid4() -> None: - # Valid UUID4 - assert is_valid_uuid4(str(uuid.uuid4())) - assert is_valid_uuid4("123e4567-e89b-4456-a456-426614174000") # Version 4 UUID - assert is_valid_uuid4("123E4567-E89B-4456-A456-426614174000") # Case doesn't matter - - # Invalid UUID4 - assert not is_valid_uuid4("not-a-uuid") - assert not is_valid_uuid4("12345678-1234-5678-1234-56781234567") # Too short - assert not is_valid_uuid4("12345678-1234-5678-1234-5678123456789") # Too long - assert not is_valid_uuid4("123e4567-e89b-1234-a456-426614174000") # Wrong version - - -def test_hash_to_uuid4_string_input() -> None: - guid = "12345678901234567890123456789012" - result = hash_to_uuid4(guid) - - # Verify it's a valid UUID - uuid_obj = uuid.UUID(result) - assert uuid_obj.version == 4 - assert uuid_obj.variant == uuid.RFC_4122 - - -def test_hash_to_uuid4_bytes_input() -> None: - guid = b"12345678901234567890123456789012" - result = hash_to_uuid4(guid.decode()) # Convert bytes to str - - # Verify it's a valid UUID - uuid_obj = uuid.UUID(result) - assert uuid_obj.version == 4 - assert uuid_obj.variant == uuid.RFC_4122 - - -def test_hash_to_uuid4_short_input() -> None: - guid = "1234567890" - result = hash_to_uuid4(guid) - - # Should still work with shorter input - uuid_obj = uuid.UUID(result) - assert uuid_obj.version == 4 - assert uuid_obj.variant == uuid.RFC_4122 - - -def test_hash_to_uuid4_deterministic() -> None: - guid = "12345678901234567890123456789012" - result1 = hash_to_uuid4(guid) - result2 = hash_to_uuid4(guid) - - # Same input should produce same output - assert result1 == result2 - - -def test_hash_to_uuid4_different_inputs() -> None: - guid1 = "12345678901234567890123456789012" - guid2 = "98765432109876543210987654321098" - result1 = hash_to_uuid4(guid1) - result2 = hash_to_uuid4(guid2) - - # Different inputs should produce different outputs - assert result1 != result2 diff --git a/tests/unit/test_version.py b/tests/unit/test_version.py index 7bc98a2b..44219301 100644 --- a/tests/unit/test_version.py +++ b/tests/unit/test_version.py @@ -1,9 +1,10 @@ +from importlib.metadata import version + import askui -from askui.telemetry.pkg_version import get_pkg_version def test_version_consistency() -> None: - package_version = get_pkg_version() + package_version = version("askui") module_version = askui.__version__ assert package_version == module_version, ( f"Version mismatch: package={package_version}, module={module_version}; "