From 21123533b73a6baf797f063a2503a15dde51e653 Mon Sep 17 00:00:00 2001 From: batuhanisildak-malwation Date: Mon, 19 Oct 2026 19:32:50 +0000 Subject: [PATCH] feat(types): add the current Public API fields and error codes Model the network outcomes, MITRE v20 fields, typed report options, plan dates and list URL. Expose the error code on every API error and add the 503 errors. --- CHANGELOG.md | 28 + docs/TYPES.md | 51 +- src/threatzone/__init__.py | 4 + src/threatzone/_exceptions.py | 168 ++++- src/threatzone/testing/_responses.py | 24 +- src/threatzone/types/__init__.py | 7 +- src/threatzone/types/common.py | 13 +- src/threatzone/types/errors.py | 8 +- src/threatzone/types/indicators.py | 8 + src/threatzone/types/me.py | 15 + src/threatzone/types/mitre.py | 16 +- src/threatzone/types/network.py | 75 +- src/threatzone/types/submissions.py | 5 + src/threatzone/types/syscalls.py | 3 + .../flows/test_flow_e2e_review_fields.py | 55 ++ tests/test_api_e2e_review_shapes.py | 695 ++++++++++++++++++ 16 files changed, 1082 insertions(+), 93 deletions(-) create mode 100644 tests/integration/flows/test_flow_e2e_review_fields.py create mode 100644 tests/test_api_e2e_review_shapes.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f72027..cd9db5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Public API fields of the 2026-10-19 release. Every new field is `None` when an older server + omits it: + - `Connection.connect_outcome` (`ConnectOutcome`: `established`, `no_response`, `refused`, + `not_captured`), `DnsQuery.answered`, and `timeshift_seconds` on `DnsQuery`, `Connection` + and `NetworkThreat`. + - `NetworkEndpoint.max_alert_severity_level` (`1` high to `4` info). + - `MitreResponse.attack_version` and `MitreResponse.domain` (`AttackDomain`), + `MitreTechniqueUsage.attempted`, `Indicator.attempted_attack_codes` and `Indicator.source`. + - `SubmissionListItem.url`, `ReportStatus.metafield_values` (`MetafieldValue`) and + `ReportStatus.execution_duration_seconds`. + - `PlanInfo.starts_at`, `PlanInfo.ends_at`, `ModuleInfo.starts_at`, `ModuleInfo.ends_at` + and `FileLimits.file_size_bytes`. + - `SyscallSearchResult.retry_after_seconds`. +- `APIError.code` holds the error code of every HTTP error, for example `WORKSPACE_INACTIVE` + (402) or `PLAN_API_ACCESS_DENIED` (403). `BadRequestError.field` gives `details.field`. +- `ServiceUnavailableError` (503, subclass of `InternalServerError`) with `retry_after`. + `QuotaUnavailableError` is now its subclass. +- `BehaviourIndexBuildingError` for `503 BEHAVIOUR_INDEX_BUILDING`. It is a subclass of + `BehaviourIndexingError` and of `ServiceUnavailableError`. +- `ApiErrorCode` adds `WORKSPACE_INACTIVE`, `PLAN_API_ACCESS_DENIED` and + `BEHAVIOUR_INDEX_BUILDING`. `DnsStatus` adds `UNKNOWN` and accepts any other string. + - Network rows (`DnsQuery`, `HttpRequest`, `Connection`, `NetworkThreat`) add `attribution`, `attribution_reason`, `process_id` and `process_image` (new base model `NetworkRowSource`, new type `NetworkAttribution`). `DnsQuery` adds `status_text`. `NetworkSummary` adds @@ -353,6 +375,12 @@ Every method above is mirrored on `AsyncThreatZone`. ### Changed +- `retry_after` on `RateLimitError` and the `503` errors falls back to + `details.retryAfterSeconds` when the header and the `retryAfter` body field are absent. +- The fake API (`threatzone.testing`) sends the new fields. +- `ApiErrorCode` no longer lists `SAMPLE_INDEX_UNAVAILABLE` and `SAMPLE_INDEX_ERROR`. The API + never sent them. + - `ReportStatus.score`, `Indicator.score`, `UrlAnalysisResponse.threat_score`, `UrlAnalysisResponse.privacy_score` and `UrlAnalysisPrivacyFactor.delta` change from `int` to `float`. Integer values from the API stay valid. A strict type checker can now report diff --git a/docs/TYPES.md b/docs/TYPES.md index 776617a..6affccf 100644 --- a/docs/TYPES.md +++ b/docs/TYPES.md @@ -22,12 +22,12 @@ Top-level submission types. Returned by `create_*_submission()`, `list_submissio | Type | Purpose | Endpoints | |------|---------|-----------| | `SubmissionCreated` | Thin envelope for a freshly queued submission. | `POST /submissions/*` | -| `SubmissionListItem` | One row in a paginated list. Carries enrichment (indicators rollup, overview). | `GET /submissions` | +| `SubmissionListItem` | One row in a paginated list. Carries enrichment (indicators rollup, overview). `url` is the submitted URL of a URL submission; `filename` and `sha256` are `None` for a URL submission. | `GET /submissions` | | `PaginatedSubmissions` | Full list envelope with `items`, `total`, `page`, `limit`, `total_pages`. | `GET /submissions` | | `Submission` | Full submission detail including `file`, `mitre_techniques`, `overview`, `score_breakdown`. `score` is the live score that the web UI shows; it is `None` for a pure URL submission until the final URL verdict. `search_by_sha256()` returns one page of these (`page`, `limit`; the server default is 20 rows). | `GET /submissions/:uuid`, `GET /submissions/search/sha256/:sha256` | | `SubmissionOverview` | Aggregated analysis status + job progress (`status`, optional `jobs`, `duration`). `duration` is the longest report run time in seconds; only `get_submission()` sends it. | Nested in `Submission` and `SubmissionListItem` | | `SubmissionIndicatorsRollup` | Severity rollup attached to a submission (`levels`, `artifact_count`). | Nested in `Submission` and `SubmissionListItem` | -| `ReportStatus` | Per-report state (`type`, `status`, `level`, `score`, `duration`, `format`, `operating_system`, `metafields`). `score` is a `float` or `None`. `metafields` is a `dict[str, str]` of the analysis options as the web UI shows them (option name to selected value); only `get_submission()` sends it. | Nested `reports[]` on `Submission` | +| `ReportStatus` | Per-report state (`type`, `status`, `level`, `score`, `duration`, `format`, `operating_system`, `metafields`). `score` is a `float` or `None`. `metafields` is a `dict[str, str]` of the analysis options as the web UI shows them (option name to selected value); only `get_submission()` sends it. `metafield_values` is the typed form keyed by the metafield key (`dict[str, MetafieldValue]`, for example `{"timeout": 120}`), and `execution_duration_seconds` is the `timeout` option of a dynamic report. Read these two in code. | Nested `reports[]` on `Submission` | | `ScoreBreakdown` | Explanation of `Submission.score`: `deciding_layer`, `verdict`, `score`, `cap`, `contributors`, `what_would_change`. Only `get_submission()` sends it. `None` for a pure URL submission, before the overview exists, and when the stored score does not come from the current score rule. | `GET /submissions/:uuid` | | `ScoreContributor` | One finding with a large effect on the score: `source`, `indicator_id`, `title`, `points`, `level`. At most ten, largest effect first. | Nested in `ScoreBreakdown.contributors` | @@ -58,7 +58,7 @@ in this group is envelope-wrapped (`items`, `total`) for server-side pagination. | Type | Purpose | Endpoint | |------|---------|----------| -| `Indicator` | A single rule hit with cross-reference IDs. Fields: `id`, `name`, `description`, `summary`, `why_matched`, `in_verdict_chain`, `category`, `level`, `score`, `score_maximum`, `pids`, `pid_gens`, `attack_codes`, `event_ids`, `syscall_line_numbers`, `author`, `rule_source`, `what_happened`, `why_it_matters`, `why_this_tier`, `not_observed`, `outcome`, `score_note`, `score_factors`, `score_result`, `score_cap`, `related`, `actor`, `target`, `t_offset_s`, `span_s`, `occurrences`, `finding_source`, `artifacts`, `dynamic_indicator_id`. | `GET /submissions/:uuid/indicators` | +| `Indicator` | A single rule hit with cross-reference IDs. Fields: `id`, `source`, `name`, `description`, `summary`, `why_matched`, `in_verdict_chain`, `category`, `level`, `score`, `score_maximum`, `pids`, `pid_gens`, `attack_codes`, `attempted_attack_codes`, `event_ids`, `syscall_line_numbers`, `author`, `rule_source`, `what_happened`, `why_it_matters`, `why_this_tier`, `not_observed`, `outcome`, `score_note`, `score_factors`, `score_result`, `score_cap`, `related`, `actor`, `target`, `t_offset_s`, `span_s`, `occurrences`, `finding_source`, `artifacts`, `dynamic_indicator_id`. | `GET /submissions/:uuid/indicators` | | `IndicatorRelated` | A behaviour event related to the row: `event_id`, `signal_id`, `text`, `t_offset_s`, `actor` (`IndicatorRelatedActor`: `image`, `pid`). | nested in `Indicator.related` | | `IndicatorActor` | The process that did the behaviour: `image`, `pid`, `pid_gen`, `lineage`. | nested in `Indicator.actor` | | `IndicatorTarget` | The object that the behaviour acted on: `kind`, `value`. | nested in `Indicator.target` | @@ -75,8 +75,8 @@ in this group is envelope-wrapped (`items`, `total`) for server-side pagination. | `ArtifactsResponse` | Artifact envelope (`items`, `total`). | same | | `ArtifactJob` | One engine analysis job of an artifact: `engine_type`, `status`, `error_code`. `get_artifact_jobs()` returns a list, oldest first. | `GET /submissions/:uuid/artifacts/:artifactId/jobs` | | `EmlAnalysis` | Parsed `.eml` artifact: `headers`, `other_headers`, `attachments`, `qr_results`, `artifact`. | `GET /submissions/:uuid/eml-analysis` | -| `MitreResponse` | `{techniques: List[str], total, items}`. Sorted, de-duplicated MITRE IDs. `items` has one `MitreTechniqueUsage` for each matched technique that is in the ATT&CK catalog of the platform, sorted by `technique_id`. A code that is not in the catalog is in `techniques` but not in `items`, so `items` can be shorter than `techniques`. `items` is `None` when an older server omits it. | `GET /submissions/:uuid/mitre` | -| `MitreTechniqueUsage` | `{technique_id, name, tactic_ids, score, score_maximum, in_verdict_chain, indicator_ids}`. `score` is the highest score (0 to 100) of the indicators that carry the technique; `score_maximum` is always `100`. `indicator_ids` are `Indicator.id` values. | nested in `MitreResponse` | +| `MitreResponse` | `{techniques: List[str], total, attack_version, domain, items}`. `attack_version` is the ATT&CK catalog version (`"20.0"`); `domain` is `AttackDomain` (`enterprise` or `mobile` for an Android report), and `items` holds only the techniques of that domain. Sorted, de-duplicated MITRE IDs. `items` has one `MitreTechniqueUsage` for each matched technique that is in the ATT&CK catalog of the platform, sorted by `technique_id`. A code that is not in the catalog is in `techniques` but not in `items`, so `items` can be shorter than `techniques`. `items` is `None` when an older server omits it. | `GET /submissions/:uuid/mitre` | +| `MitreTechniqueUsage` | `{technique_id, name, tactic_ids, score, score_maximum, in_verdict_chain, attempted, indicator_ids}`. `tactic_ids` lists every tactic of the technique in matrix order. `attempted` is `True` when every carrying indicator marks the technique attempted (a request that the sandbox did not confirm). `score` is the highest score (0 to 100) of the indicators that carry the technique; `score_maximum` is always `100`. `indicator_ids` are `Indicator.id` values. | nested in `MitreResponse` | | `StaticScanResponse` | Static report singleton, discriminated by `report_format`. Format-specific keys stay in `.model_extra`. | `GET /submissions/:uuid/static-scan` | | `StaticScanFileInfo` | `{md5, sha1, sha256, ssdeep, mime_type, file_type, entropy, filesize, real_filename, real_sha256}`. | nested in `StaticScanResponse` | | `StaticScanOneNoteEmbeddedFile` | One `oneNoteEmbeddedFiles[]` entry of an `onenote` report. Read it with `StaticScanResponse.one_note_embedded_files`. | same | @@ -183,31 +183,31 @@ report. Every endpoint is guarded by `ReportStatusGuard('dynamic')`; read the co | `BehaviourEventSort` / `BehaviourFacetField` / `BehaviourFieldFilterKey` | Parameter values of `get_behaviours(sort=...)` (`eventId`, `time`), `get_behaviour_facets(field)` (`operation`, `type`, `processName`, `function`, `subject`) and the `fields` keys (`path`, `key`, `mutex`, `cmd`, `status`, `value`, …). | parameters | | `BehaviourOs` | Deprecated compatibility alias; the endpoint derives the OS from the submission. | same | | `SyscallsResponse` | `{items: List[str], total, page, limit, total_pages, first_line}`. `items` are raw unparsed log lines. `items[i]` is line `first_line + i`, the same number as `BehaviourEvent.syscall_line_number`. `first_line` is `None` when `items` is empty. `page`, `limit`, `total_pages` and `first_line` are `None` when an older server omits them. | `GET /submissions/:uuid/syscalls` | -| `SyscallSearchResult` | `{lines, next_from_line, scanned_to, total_count, done, truncated, busy}`. `lines` are `SyscallSearchLine` (`line`, `raw`). Send `next_from_line` as `after_line` to get the next page; it is `None` when the scan read the last line. `busy` is `True` when the server did not scan; then `total_count` is `None`. | `GET /submissions/:uuid/syscalls/search` | +| `SyscallSearchResult` | `{lines, next_from_line, scanned_to, total_count, done, truncated, busy, retry_after_seconds}`. `lines` are `SyscallSearchLine` (`line`, `raw`). Send `next_from_line` as `after_line` to get the next page; it is `None` when the scan read the last line. `busy` is `True` when the server did not scan; then `total_count` is `None` and `retry_after_seconds` gives the wait. | `GET /submissions/:uuid/syscalls/search` | | `NetworkSummary` | `{dns_count, http_count, tcp_count, udp_count, threat_count, pcap_available, exfil_request_count, counts_available, availability, source_scope, windows_background_count, https_pcap_available, has_https_inspection, unattributed_pcap_available, attribution_counts, unattributed_counts, submission_only_alerts}`. `exfil_request_count` gives the entries of `get_network_exfil_requests()`; it is not set to 0 when `counts_available` is `False`. `windows_background_count` gives the rows that the counts and lists omit by default. Legacy reports send `counts_available=True`, `availability=None`, `source_scope=None`; all three are `None` when an older server omits them. `https_pcap_available` tells that `download_https_pcap()` has a file. `unattributed_pcap_available` is `True` when `download_pcap()` returns the full capture of a `submission_unattributed` report. `unattributed_counts` and `submission_only_alerts` are `None` when the report has none. Every field after `windows_background_count` is `None` when an older server omits it. | `GET /submissions/:uuid/network/summary` | | `NetworkAvailability` | `{state, reason, coverage}`. Network evidence state of a Windows V2 report. | nested in `NetworkSummary` | | `NetworkAttributionCounts` / `NetworkSourceCounts` | `{dns, http, tcp, udp, threats}`, each `{sample, sample_child, sensor_only, windows_background}`. Rows of each list by source, also the `windows_background` rows. Rows without a stored source count as `sample`. | nested in `NetworkSummary` | | `NetworkUnattributedCounts` | `{dns, http, tcp, udp, threats}`. Records of the sensor summary of an older report that no sample process is linked to. The lists do not return them. | nested in `NetworkSummary` | | `NetworkSubmissionOnlyAlerts` | `{scope, source, total, omitted, rows}`. Suricata alerts outside the remote addresses of the sample (`source_scope == "submission_ip_scoped"`). `rows` are at most 100 `NetworkSubmissionOnlyAlertRow` (`line_number`, `sid`, `signature`, `severity`, `protocol`, `source_ip`, `destination_ip`). `get_network_threats()` does not return them. | nested in `NetworkSummary` | | `NetworkRowSource` | `{attribution, attribution_reason, process_id, process_image}` on every network row. `attribution` is `NetworkAttribution` (`sample`, `sample_child`, `sensor_only`, `windows_background`, or a new value). The server sends `sample` for rows of older reports; each field is `None` when an older server omits it. | base of the network rows | -| `DnsQuery` | `{id, host, type, status, status_text, records, timeshift, source_scope}` plus the `NetworkRowSource` fields. `status_text` keeps the sensor text when `status` is `TIMEOUT`. | `GET /submissions/:uuid/network/dns` | +| `DnsQuery` | `{id, host, type, status, status_text, records, answered, timeshift, timeshift_seconds, source_scope}` plus the `NetworkRowSource` fields. `answered` is `False` for a query that got no answer. `status` is `TIMEOUT` for such a query and `UNKNOWN` for a non-standard sensor text; `status_text` keeps the sensor text. | `GET /submissions/:uuid/network/dns` | | `HttpRequest` | `{id, host, ip, port, country, source_scope}` plus the `NetworkRowSource` fields — slimmed-down shape. Read the method, URL, headers and bodies with `get_http_request_packets(uuid, row.id)`. | `GET /submissions/:uuid/network/http` | -| `Connection` | `{id, protocol, source_ip, source_port, destination_ip, destination_port, domain, ptr_domain, asn, country, packets, app_proto, hostname, pkts_to_server, pkts_to_client, bytes_to_server, bytes_to_client, packets_total, packets_omitted, timeshift, source_scope}` plus the `NetworkRowSource` fields. Used for both TCP and UDP. `domain` is the best host name of `destination_ip` (DNS answer, TLS SNI, HTTP `Host`, reverse DNS, then sensor host name); `ptr_domain` is the reverse DNS name and can differ. `app_proto`, `hostname` and the `pkts_*`/`bytes_*` counters come from the live network sensor. `packets_omitted` counts the capture packets that the report packet limit left out. Each new field is `None` for rows of older reports and when an older server omits it. | `GET /submissions/:uuid/network/tcp`, `GET /submissions/:uuid/network/udp` | +| `Connection` | `{id, protocol, source_ip, source_port, destination_ip, destination_port, domain, ptr_domain, asn, country, packets, app_proto, hostname, pkts_to_server, pkts_to_client, bytes_to_server, bytes_to_client, packets_total, packets_omitted, connect_outcome, timeshift, timeshift_seconds, source_scope}` plus the `NetworkRowSource` fields. Used for both TCP and UDP. `connect_outcome` is `ConnectOutcome`: `established`, `no_response`, `refused` or `not_captured` (the process monitor saw the connect, but the capture holds no flow). `domain` is the best host name of `destination_ip` (DNS answer, TLS SNI, HTTP `Host`, reverse DNS, then sensor host name); `ptr_domain` is the reverse DNS name and can differ. `app_proto`, `hostname` and the `pkts_*`/`bytes_*` counters come from the live network sensor. `packets_omitted` counts the capture packets that the report packet limit left out. Each new field is `None` for rows of older reports and when an older server omits it. | `GET /submissions/:uuid/network/tcp`, `GET /submissions/:uuid/network/udp` | | `ConnectionPackets` | `{sent, received, empty}`. | nested | -| `NetworkThreat` | Suricata alert: `signature`, `description`, `severity`, `severity_level`, `protocol`, `app_proto`, `source_ip`, `source_port`, `destination_ip`, `destination_port`, `timeshift`, `metadata`, `details`, optional `tls`, `capture`, `source_scope`, plus the `NetworkRowSource` fields. | `GET /submissions/:uuid/network/threats` | +| `NetworkThreat` | Suricata alert: `signature`, `description`, `severity`, `severity_level`, `protocol`, `app_proto`, `source_ip`, `source_port`, `destination_ip`, `destination_port`, `timeshift`, `timeshift_seconds`, `metadata`, `details`, optional `tls`, `capture`, `source_scope`, plus the `NetworkRowSource` fields. | `GET /submissions/:uuid/network/threats` | | `NetworkThreatTls` | TLS fingerprint block: `version`, `sni`, `ja3_hash`, `ja3s_hash`, `ja3_string`, `ja3s_string`. Present iff `app_proto == "TLS"`. | nested in `NetworkThreat` | | `NetworkExfilRequest` | Chat bot and webhook requests from the decrypted HTTPS capture: `api`, `kind`, `host`, `method`, `api_method`, `target_id`, `masked_target`, `request_bytes`, `reply_status`, `request_count`, `capture`, `first_seen`, `score`, `tier`, `process_id`, `process_image`, `text`. `masked_target` is the request target as sent, with the full bot token or webhook secret; old reports show a shortened token. Every field is `None` when the server omits it. | `GET /submissions/:uuid/network/exfil-requests` | | `TotalCountList[T]` | Return type of the six network list methods. It is a plain `list` of the rows with one more attribute: `total_count`, the `X-Total-Count` header (rows that match the request without `skip` and `limit`, with the same filters). `total_count` is `None` when the server does not send the header. More pages exist while `skip + len(page) < total_count`. | network lists | | `NetworkHttpPackets` | `{items, truncated, source_scope}`. `items` are at most 200 `NetworkHttpPacket` values (`timeshift`, `url`, `method`, `destination_ip`, `destination_port`, `port_known`, `headers`, `body`, `response`). `response` is a `NetworkHttpPacketResponse` (`timeshift`, `destination_ip`, `destination_port`, `headers`, `status_code`, `body`) or `None`. When `port_known` is `False` (a sensor summary record of an older report), the ports are 0 and the headers and bodies are `None`. | `GET /submissions/:uuid/network/http/:requestId/packets` | | `NetworkFlowPackets` | `{protocol, source_ip, source_port, destination_ip, destination_port, items, truncated, source_scope}`. `items` are at most 200 `NetworkFlowPacket` values (`type`, `data`). `type` is `NetworkPacketDirection` (`send`, `receive`, or a new value). | `GET /submissions/:uuid/network/tcp/:connectionId/packets`, `.../udp/:connectionId/packets` | | `NetworkPacketData` | `{trimmed, sha256, md5, sha1, ssdeep, data}`. `data` is `bytes`: the SDK decodes the base64 text of the API. `model_dump(mode="json")` encodes it as base64 again. `trimmed` is `True` when the sandbox stored only the first part of the payload. The hashes cover the stored bytes and are `None` when the sandbox did not compute them. The payload is not masked. | nested in the packet models | -| `NetworkEndpoints` | `{items, total, truncated}`. `items` are `NetworkEndpoint` values (`key`, `host`, `ips`, `ports`, `protocols`, `source`, `first_timeshift`, `last_timeshift`, `request_count`, `alert_count`, `max_alert_severity`, `process_ids`, `attribution_note`), sample endpoints first. `total` is the count before `limit`, a lower limit when `truncated` is `True`. Send `key` as `host` to `get_network_host_flows()`. | `GET /submissions/:uuid/network/endpoints` | +| `NetworkEndpoints` | `{items, total, truncated}`. `items` are `NetworkEndpoint` values (`key`, `host`, `ips`, `ports`, `protocols`, `source`, `first_timeshift`, `last_timeshift`, `request_count`, `alert_count`, `max_alert_severity`, `max_alert_severity_level`, `process_ids`, `attribution_note`), sample endpoints first. `max_alert_severity_level` is the highest alert severity as a number, `1` (high) to `4` (info); `max_alert_severity` is the deprecated text form. `total` is the count before `limit`, a lower limit when `truncated` is `True`. Send `key` as `host` to `get_network_host_flows()`. | `GET /submissions/:uuid/network/endpoints` | | `NetworkHostFlows` | `{flows, totals, alerts, page_info, truncated}`. `flows` are `NetworkHostFlow` values (`protocol`, `remote_ip`, `remote_port`, `local_port`, `count`, `bytes_sent`, `bytes_received`, `first_timeshift`, `last_timeshift`, `process_ids`, `methods`, `sample_paths`), the busiest first. `totals` is `NetworkHostFlowTotals` (`dns_queries`, `http_requests`, `tcp_flows`, `udp_flows`). `alerts` are up to 20 `NetworkHostAlert` groups (`signature`, `severity`, `count`, `first_timeshift`), the most severe first; it is empty when an older server omits it. `page_info` is `NetworkHostFlowsPageInfo` (`end_cursor`, `has_next_page`). | `GET /submissions/:uuid/network/endpoints/:host/flows` | **Discriminative fields.** - `DnsQuery.type` is `DnsRecordType` (16-value literal: `A`, `AAAA`, `CNAME`, `MX`, …). -- `DnsQuery.status` is `DnsStatus` (DNS RCODE literal, e.g. `NOERROR`, `NXDOMAIN`, `TIMEOUT`). +- `DnsQuery.status` is `DnsStatus` (DNS RCODE literal, e.g. `NOERROR`, `NXDOMAIN`, `TIMEOUT`, `UNKNOWN`, or any other string). - `NetworkAvailability.state` is `NetworkAvailabilityState` (`disabled`, `unavailable`, `verified_zero`, `observed`); `coverage` is `NetworkCoverage` (`none`, `partial`, `complete`). When `NetworkSummary.counts_available` is `False`, all counts are 0 and do not mean the sample had no network activity. - `source_scope` is `NetworkSourceScope` (`sample_attributed`, `submission_ip_scoped`, `submission_unattributed`, `none`) or `None` on legacy reports. Unknown values from a newer server are kept as plain strings. - The list endpoints send the `X-Network-Evidence-Availability` response header on Windows V2. The SDK does not surface it. Read `get_network_summary().availability` instead. The SDK surfaces only the `X-Total-Count` header, as `TotalCountList.total_count`. @@ -418,11 +418,11 @@ User, workspace, plan, and quota types. Returned by `get_user_info()`. | `UserInfoDetails` | Nested user details: `email`, `full_name`, `workspace`, `limits_count`. | | `WorkspaceBasicInfo` | `{name, alias, private, type}`. `type` is `Literal["personal", "organization"]`. | | `LimitsCount` | Current usage: `api_request_count`, `daily_submission_count`, `concurrent_submission_count`, `ai_report_count`, `ai_report_resets_at`. `ai_report_count` counts the AI report quota units used since 00:00 UTC; a report in progress counts as used. `ai_report_resets_at` is the next 00:00 UTC. | -| `PlanInfo` | `{plan_name, start_time, end_time, subs_time, file_limits, submission_limits, permissions}`. | +| `PlanInfo` | `{plan_name, start_time, end_time, starts_at, ends_at, subs_time, file_limits, submission_limits, permissions}`. `start_time` and `end_time` are display text; read `starts_at` and `ends_at` (`datetime` or `None`) in code. | | `PlanPermissions` | `{download_sample, html_report, stix_report, pdf_report, download_report, ai_report}`. A `False` value means that the matching routes return 403. | -| `FileLimits` | `{extensions, file_size}`. | +| `FileLimits` | `{extensions, file_size, file_size_bytes}`. `file_size` is display text; read `file_size_bytes` in code. | | `SubmissionLimits` | `{api_limit, daily_limit, concurrent_limit, ai_report_limit}`. `ai_report_limit` is `0` when the plan has no AI report. | -| `ModuleInfo` | One enabled module: `{module_id, module_name, start_time, end_time}`. | +| `ModuleInfo` | One enabled module: `{module_id, module_name, start_time, end_time, starts_at, ends_at}`. | Check `account.user_info.limits_count.api_request_count` to decide whether to back off before hammering rate-limited endpoints. @@ -439,20 +439,22 @@ Typed error surface. Import from the top-level `threatzone` module or from | Class | Kind | Key Fields | |-------|------|------------| | `ThreatZoneError` | Base class for every SDK error. | `message` | -| `APIError` | Base class for HTTP errors. | `status_code`, `response`, `body` | +| `APIError` | Base class for HTTP errors. | `status_code`, `code`, `response`, `body` | | `AuthenticationError` (401) | Invalid/missing API key. | inherits from `APIError` | -| `PaymentRequiredError` (402) | Workspace subscription inactive. | inherits from `APIError` | -| `PermissionDeniedError` (403) | Plan permission or module missing (`FORBIDDEN`), or an unlicensed self-hosted deployment (`LICENSE_INVALID`). | inherits from `APIError` | +| `PaymentRequiredError` (402) | `WORKSPACE_INACTIVE`: the workspace is not active. Renew the plan or contact the workspace owner. | `code` | +| `PermissionDeniedError` (403) | Plan permission or module missing (`FORBIDDEN`), a plan without API access (`PLAN_API_ACCESS_DENIED`, a retry does not help), or an unlicensed self-hosted deployment (`LICENSE_INVALID`). | `code` | | `NotFoundError` (404) | `NOT_FOUND` for a missing submission and for a private submission of another workspace. Route codes such as `ARTIFACT_NOT_FOUND` and `MEDIA_NOT_FOUND`. | inherits from `APIError` | -| `BadRequestError` (400) | Validation failure (`INVALID_QUERY_PARAM`, also for a malformed UUID or ID). | inherits from `APIError` | -| `RateLimitError` (429) | `RATE_LIMIT_EXCEEDED` (heavy-route limit, monthly API request limit), `SUBMISSION_LIMIT_EXCEEDED` (daily or concurrent submission limit) or `AI_REPORT_QUOTA_EXCEEDED`. | `code`, `retry_after` (from the `Retry-After` header; else, or when the header is not a number, the `retryAfter` body field) | +| `BadRequestError` (400) | Validation failure (`INVALID_QUERY_PARAM`, also for a malformed UUID or ID). | `code`, `field` (`details.field`, the first field that failed) | +| `RateLimitError` (429) | `RATE_LIMIT_EXCEEDED` (heavy-route limit, monthly API request limit), `SUBMISSION_LIMIT_EXCEEDED` (daily or concurrent submission limit) or `AI_REPORT_QUOTA_EXCEEDED`. | `code`, `retry_after` (from the `Retry-After` header, else the `retryAfter` body field, else `details.retryAfterSeconds`) | | `ReportUnavailableError` (409) | Required report not yet available. | `code`, `submission_uuid`, `required_report`, `current_status`, `available_reports` | -| `BehaviourIndexingError` (409) | `BEHAVIOUR_INDEXING`: the behaviour index is not ready. Subclass of `ReportUnavailableError`. | `code`, `retry_after` | +| `BehaviourIndexingError` (409) | `BEHAVIOUR_INDEXING` from an older server: the behaviour index is not ready. Subclass of `ReportUnavailableError`. | `code`, `retry_after` | +| `BehaviourIndexBuildingError` (503) | `BEHAVIOUR_INDEX_BUILDING`: the server builds the behaviour index and the direct read cannot answer. Subclass of `BehaviourIndexingError` and `ServiceUnavailableError`. | `code`, `retry_after` | | `BehaviourTooLargeError` (413) | `BEHAVIOUR_TOO_LARGE`: the behaviour file or the index scan is above the read limit. Use `download_behaviour()`. | `code`, `size_bytes`, `max_bytes`, `download_route` | | `BehaviourUnreadableError` (502) | `BEHAVIOUR_UNREADABLE`: the stored behaviour file is not a valid JSON array. Subclass of `InternalServerError`. | `code` | | `YaraRulePendingError` (202) | YARA rule generation still in progress. | `retry_after` | | `PdfReportPendingError` (202) | The PDF report is not ready: `download_pdf_report(wait=False)` got 202, or `download_pdf_report(wait=True)` reached its timeout. | `retry_after` | -| `QuotaUnavailableError` (503) | `QUOTA_UNAVAILABLE`: the server cannot check the submission quota, so it did not create the submission. Subclass of `InternalServerError`. | `code`, `retry_after` | +| `ServiceUnavailableError` (503) | Any other `503`, for example `STORAGE_UNAVAILABLE`. Subclass of `InternalServerError`. | `code`, `retry_after` | +| `QuotaUnavailableError` (503) | `QUOTA_UNAVAILABLE`: the server cannot check the submission quota, so it did not create the submission. Subclass of `ServiceUnavailableError`. | `code`, `retry_after` | | `InternalServerError` (5xx) | Server-side failure. | inherits from `APIError` | | `TimeoutError` | HTTP request timed out. | `message` | | `ConnectionError` | Network-level failure. | `message` | @@ -462,15 +464,16 @@ Typed error surface. Import from the top-level `threatzone` module or from **ApiErrorCode values:** -`INVALID_QUERY_PARAM`, `UNAUTHORIZED`, `LICENSE_INVALID`, `FORBIDDEN`, `NOT_FOUND`, +`INVALID_QUERY_PARAM`, `UNAUTHORIZED`, `WORKSPACE_INACTIVE`, `PLAN_API_ACCESS_DENIED`, +`LICENSE_INVALID`, `FORBIDDEN`, `NOT_FOUND`, `METHOD_NOT_ALLOWED`, `CONFLICT`, `ANNOTATION_PIN_LIMIT_REACHED`, `ANNOTATION_PIN_NOT_FOUND`, `ARTIFACT_NOT_FOUND`, `SAMPLE_NOT_AVAILABLE`, `MEDIA_NOT_FOUND`, `NETWORK_ROW_NOT_FOUND`, `URL_SCORE_EVIDENCE_NOT_FOUND`, `URL_BROWSER_SECTION_NOT_FOUND`, `URL_BROWSER_SECTION_UNAVAILABLE`, `DYNAMIC_REPORT_UNAVAILABLE`, `STATIC_REPORT_UNAVAILABLE`, `CDR_REPORT_UNAVAILABLE`, `URL_ANALYSIS_REPORT_UNAVAILABLE`, `RATE_LIMIT_EXCEEDED`, `SUBMISSION_LIMIT_EXCEEDED`, `QUOTA_UNAVAILABLE`, `INTERNAL_ERROR`, `STORAGE_UNAVAILABLE`, -`BEHAVIOUR_INDEXING`, `BEHAVIOUR_TOO_LARGE`, `BEHAVIOUR_UNREADABLE`, `SAMPLE_INDEX_UNAVAILABLE`, -`SAMPLE_INDEX_ERROR`, `SESSION_SERVICE_UNAVAILABLE`, `SESSION_NOT_FOUND`, `AI_REPORT_NOT_FOUND`, +`BEHAVIOUR_INDEX_BUILDING`, `BEHAVIOUR_INDEXING` (older servers), `BEHAVIOUR_TOO_LARGE`, +`BEHAVIOUR_UNREADABLE`, `SESSION_SERVICE_UNAVAILABLE`, `SESSION_NOT_FOUND`, `AI_REPORT_NOT_FOUND`, `AI_REPORTS_DISABLED`, `AI_REPORT_NOT_FINALIZED`, `AI_REPORT_IN_PROGRESS`, `AI_REPORT_QUOTA_EXCEEDED`, `AI_REPORT_QUEUE_UNAVAILABLE`. diff --git a/src/threatzone/__init__.py b/src/threatzone/__init__.py index 397f3c7..f9249c8 100644 --- a/src/threatzone/__init__.py +++ b/src/threatzone/__init__.py @@ -16,6 +16,7 @@ APIError, AuthenticationError, BadRequestError, + BehaviourIndexBuildingError, BehaviourIndexingError, BehaviourTooLargeError, BehaviourUnreadableError, @@ -28,6 +29,7 @@ QuotaUnavailableError, RateLimitError, ReportUnavailableError, + ServiceUnavailableError, ThreatZoneError, TimeoutError, YaraRulePendingError, @@ -211,6 +213,7 @@ "BadRequestError", "RateLimitError", "QuotaUnavailableError", + "ServiceUnavailableError", "InternalServerError", "TimeoutError", "ConnectionError", @@ -219,6 +222,7 @@ "PdfReportPendingError", "ReportUnavailableError", "BehaviourIndexingError", + "BehaviourIndexBuildingError", "BehaviourTooLargeError", "BehaviourUnreadableError", # Types - Common diff --git a/src/threatzone/_exceptions.py b/src/threatzone/_exceptions.py index a434a92..1ce188f 100644 --- a/src/threatzone/_exceptions.py +++ b/src/threatzone/_exceptions.py @@ -18,7 +18,13 @@ def __init__(self, message: str) -> None: class APIError(ThreatZoneError): - """Base exception for API errors with HTTP response details.""" + """Base exception for API errors with HTTP response details. + + Attributes: + code: The machine-readable error code from the response envelope, for example + ``WORKSPACE_INACTIVE`` or ``PLAN_API_ACCESS_DENIED``. ``None`` when the + response has no JSON error envelope. + """ def __init__( self, @@ -27,11 +33,13 @@ def __init__( status_code: int, response: httpx.Response | None = None, body: dict[str, Any] | None = None, + code: str | None = None, ) -> None: + super().__init__(message) self.status_code = status_code self.response = response self.body = body - super().__init__(message) + self.code: str | None = code if code is not None else _body_code(body) def __str__(self) -> str: return f"{self.message} (status_code={self.status_code})" @@ -42,11 +50,19 @@ class AuthenticationError(APIError): class PaymentRequiredError(APIError): - """Raised when workspace subscription is inactive (HTTP 402).""" + """Raised when the workspace is not active (HTTP 402, ``WORKSPACE_INACTIVE``). + + Renew the plan or contact the workspace owner. + """ class PermissionDeniedError(APIError): - """Raised when access is denied or module not available (HTTP 403).""" + """Raised when access is denied (HTTP 403). + + ``code`` is ``FORBIDDEN`` for a missing plan permission or module, + ``PLAN_API_ACCESS_DENIED`` when the plan has no API access (a retry does not help), + or ``LICENSE_INVALID``. + """ class NotFoundError(APIError): @@ -54,7 +70,14 @@ class NotFoundError(APIError): class BadRequestError(APIError): - """Raised when the request is invalid (HTTP 400).""" + """Raised when the request is invalid (HTTP 400, ``INVALID_QUERY_PARAM``).""" + + @property + def field(self) -> str | None: + """Path of the first field that failed validation (``details.field``), for example + ``limit``. A nested field uses dots. ``None`` when the server does not send it.""" + value = _details(self.body).get("field") + return value if isinstance(value, str) else None class RateLimitError(APIError): @@ -81,9 +104,8 @@ def __init__( retry_after: float | None = None, code: str | None = None, ) -> None: + super().__init__(message, status_code=status_code, response=response, body=body, code=code) self.retry_after = retry_after - self.code = code - super().__init__(message, status_code=status_code, response=response, body=body) def __str__(self) -> str: base = super().__str__() @@ -125,12 +147,11 @@ def __init__( current_status: str | None = None, available_reports: list[str] | None = None, ) -> None: - self.code = code + super().__init__(message, status_code=status_code, response=response, body=body, code=code) self.submission_uuid = submission_uuid self.required_report = required_report self.current_status = current_status self.available_reports = available_reports or [] - super().__init__(message, status_code=status_code, response=response, body=body) def __str__(self) -> str: base = super().__str__() @@ -140,13 +161,13 @@ def __str__(self) -> str: class BehaviourIndexingError(ReportUnavailableError): - """Raised when the behaviour index of a submission is not ready (HTTP 409). + """Raised when the behaviour index of a submission is not ready. - The routes that read the behaviour index raise it with the code - ``BEHAVIOUR_INDEXING``: ``get_behaviours()`` with an index parameter, - ``get_behaviour_summary()`` and ``get_behaviour_facets()``. The public API - never builds the index. The index is built when the Behaviour view of the - report is opened. Retry after ``retry_after`` seconds, or call + The routes that read the behaviour index raise it: ``get_behaviours()`` with an + index parameter, ``get_behaviour_summary()``, ``get_behaviour_facets()`` and + ``get_behaviour_matches()``. Current servers return ``503 BEHAVIOUR_INDEX_BUILDING``, + which raises the subclass ``BehaviourIndexBuildingError``. Older servers return + ``409 BEHAVIOUR_INDEXING``. Retry after ``retry_after`` seconds, or call ``get_behaviours()`` without the index parameters. Attributes: @@ -163,8 +184,8 @@ def __init__( code: str | None = "BEHAVIOUR_INDEXING", retry_after: float | None = None, ) -> None: - self.retry_after = retry_after super().__init__(message, status_code=status_code, response=response, body=body, code=code) + self.retry_after = retry_after def __str__(self) -> str: base = super().__str__() @@ -199,11 +220,10 @@ def __init__( max_bytes: int | None = None, download_route: str | None = None, ) -> None: - self.code = code + super().__init__(message, status_code=status_code, response=response, body=body, code=code) self.size_bytes = size_bytes self.max_bytes = max_bytes self.download_route = download_route - super().__init__(message, status_code=status_code, response=response, body=body) class YaraRulePendingError(APIError): @@ -218,8 +238,8 @@ def __init__( body: dict[str, Any] | None = None, retry_after: float | None = None, ) -> None: - self.retry_after = retry_after super().__init__(message, status_code=status_code, response=response, body=body) + self.retry_after = retry_after def __str__(self) -> str: base = super().__str__() @@ -245,8 +265,8 @@ def __init__( body: dict[str, Any] | None = None, retry_after: float | None = None, ) -> None: - self.retry_after = retry_after super().__init__(message, status_code=status_code, response=response, body=body) + self.retry_after = retry_after def __str__(self) -> str: base = super().__str__() @@ -259,7 +279,74 @@ class InternalServerError(APIError): """Raised when a server error occurs (HTTP 5xx).""" -class QuotaUnavailableError(InternalServerError): +class ServiceUnavailableError(InternalServerError): + """Raised when the server is busy or a dependency is down (HTTP 503). + + Every ``503`` of the API has a ``Retry-After`` header and ``details.retryAfterSeconds``. + The SDK retries idempotent requests on ``503`` when the wait is short. Retry other + requests after ``retry_after`` seconds. + + Attributes: + retry_after: Seconds to wait before a retry. The SDK reads the ``Retry-After`` + header, then the ``retryAfter`` body field, then ``details.retryAfterSeconds``. + """ + + def __init__( + self, + message: str, + *, + status_code: int = 503, + response: httpx.Response | None = None, + body: dict[str, Any] | None = None, + code: str | None = None, + retry_after: float | None = None, + ) -> None: + super().__init__(message, status_code=status_code, response=response, body=body, code=code) + self.retry_after = retry_after + + def __str__(self) -> str: + base = super().__str__() + if self.retry_after is not None: + return f"{base} (retry_after={self.retry_after}s)" + return base + + +class BehaviourIndexBuildingError(BehaviourIndexingError, ServiceUnavailableError): + """Raised when the behaviour index is being built (HTTP 503, ``BEHAVIOUR_INDEX_BUILDING``). + + The behaviour index routes ask the platform to build a missing index and answer from a + direct read of the behaviour file. When that read cannot answer the request, they return + this error. Retry after ``retry_after`` seconds. It is a ``BehaviourIndexingError``, so + code that handles the older ``409 BEHAVIOUR_INDEXING`` also handles it. + """ + + def __init__( + self, + message: str, + *, + status_code: int = 503, + response: httpx.Response | None = None, + body: dict[str, Any] | None = None, + code: str | None = "BEHAVIOUR_INDEX_BUILDING", + retry_after: float | None = None, + ) -> None: + super().__init__( + message, + status_code=status_code, + response=response, + body=body, + code=code, + retry_after=retry_after, + ) + + def __str__(self) -> str: + base = APIError.__str__(self) + if self.retry_after is not None: + return f"{base} (retry_after={self.retry_after}s)" + return base + + +class QuotaUnavailableError(ServiceUnavailableError): """Raised when the server cannot check the submission quota (HTTP 503). Maps the code ``QUOTA_UNAVAILABLE``. The create routes and @@ -283,15 +370,14 @@ def __init__( code: str | None = "QUOTA_UNAVAILABLE", retry_after: float | None = None, ) -> None: - self.code = code - self.retry_after = retry_after - super().__init__(message, status_code=status_code, response=response, body=body) - - def __str__(self) -> str: - base = super().__str__() - if self.retry_after is not None: - return f"{base} (retry_after={self.retry_after}s)" - return base + super().__init__( + message, + status_code=status_code, + response=response, + body=body, + code=code, + retry_after=retry_after, + ) class BehaviourUnreadableError(InternalServerError): @@ -312,8 +398,7 @@ def __init__( body: dict[str, Any] | None = None, code: str | None = "BEHAVIOUR_UNREADABLE", ) -> None: - self.code = code - super().__init__(message, status_code=status_code, response=response, body=body) + super().__init__(message, status_code=status_code, response=response, body=body, code=code) class TimeoutError(ThreatZoneError): @@ -367,8 +452,14 @@ def _header_seconds(value: str | None) -> float | None: def _retry_after(response: httpx.Response, body: dict[str, Any] | None) -> float | None: - seconds = _header_seconds(response.headers.get("retry-after")) - return seconds if seconds is not None else _body_number(body, "retryAfter") + for seconds in ( + _header_seconds(response.headers.get("retry-after")), + _body_number(body, "retryAfter"), + _detail_number(body, "retryAfterSeconds"), + ): + if seconds is not None: + return seconds + return None def raise_for_status(response: httpx.Response) -> None: @@ -470,12 +561,17 @@ def raise_for_status(response: httpx.Response) -> None: retry_after=_retry_after(response, body), code=_body_code(body), ) - elif status_code == 503 and _body_code(body) == "QUOTA_UNAVAILABLE": - raise QuotaUnavailableError( + elif status_code == 503: + unavailable_class: type[ServiceUnavailableError] = { + "QUOTA_UNAVAILABLE": QuotaUnavailableError, + "BEHAVIOUR_INDEX_BUILDING": BehaviourIndexBuildingError, + }.get(_body_code(body) or "", ServiceUnavailableError) + raise unavailable_class( message, status_code=status_code, response=response, body=body, + code=_body_code(body), retry_after=_retry_after(response, body), ) elif status_code == 502 and _body_code(body) == "BEHAVIOUR_UNREADABLE": diff --git a/src/threatzone/testing/_responses.py b/src/threatzone/testing/_responses.py index 5170714..98bde1b 100644 --- a/src/threatzone/testing/_responses.py +++ b/src/threatzone/testing/_responses.py @@ -238,6 +238,7 @@ def _submission_list_item_payload(state: SubmissionState) -> dict[str, Any]: "uuid": state.uuid, "filename": state.filename if state.type == "file" else None, "sha256": state.sha256 if state.type == "file" else None, + "url": state.url if state.type == "url" else None, "level": state.level, "type": state.type, "private": state.private, @@ -341,6 +342,7 @@ def build_indicators_response( items.append( { "id": f"ind-{idx}", + "source": "dynamic_scan", "name": f"Indicator {seed.attack_code}", "description": f"Triggered MITRE technique {seed.attack_code}", "category": ["default"], @@ -597,6 +599,7 @@ def _mitre_items(state: SubmissionState) -> list[dict[str, Any]]: "score": max((_indicator_score(seed) for _, seed in carriers), default=0), "scoreMaximum": 100, "inVerdictChain": any(seed.level == "malicious" for _, seed in carriers), + "attempted": False, "indicatorIds": [f"ind-{idx}" for idx, _ in carriers], } ) @@ -610,6 +613,8 @@ def build_mitre_response(state: SubmissionState) -> MitreResponse: { "techniques": list(state.mitre_techniques), "total": len(state.mitre_techniques), + "attackVersion": "20.0", + "domain": "enterprise", "items": _mitre_items(state), } ) @@ -1376,6 +1381,7 @@ def build_dns_queries(state: SubmissionState) -> list[DnsQuery]: "type": "A", "status": "NOERROR", "records": ["192.0.2.10"], + "answered": True, "timeshift": 0.5, }, { @@ -1384,6 +1390,7 @@ def build_dns_queries(state: SubmissionState) -> list[DnsQuery]: "type": "A", "status": "NOERROR", "records": ["198.51.100.1"], + "answered": True, "timeshift": 1.0, }, ] @@ -1433,6 +1440,7 @@ def _build_connection_payload(proto: str, identifier: str, ip: str, port: int) - "bytesToClient": 6120, "packetsTotal": 30, "packetsOmitted": 0, + "connectOutcome": "established", "timeshift": 0.5, } @@ -1623,6 +1631,7 @@ def _endpoint_row( "requestCount": request_count, "alertCount": alert_count, "maxAlertSeverity": max_alert_severity, + "maxAlertSeverityLevel": int(max_alert_severity) if max_alert_severity else None, "processIds": [], "attributionNote": None, } @@ -2411,12 +2420,15 @@ def build_user_info() -> UserInfo: }, "plan": { "planName": "Enterprise", - "startTime": "2024-01-01T00:00:00Z", - "endTime": "2099-01-01T00:00:00Z", + "startTime": "1 January 2024", + "endTime": "1 January 2099", + "startsAt": "2024-01-01T00:00:00.000Z", + "endsAt": "2099-01-01T00:00:00.000Z", "subsTime": "monthly", "fileLimits": { "extensions": ["exe", "dll", "pdf"], - "fileSize": "100MB", + "fileSize": "100.00 MB", + "fileSizeBytes": 104857600, }, "submissionLimits": { "apiLimit": 1000, @@ -2437,8 +2449,10 @@ def build_user_info() -> UserInfo: { "moduleId": "dynamic", "moduleName": "Dynamic Analysis", - "startTime": "2024-01-01T00:00:00Z", - "endTime": "2099-01-01T00:00:00Z", + "startTime": "1 January 2024", + "endTime": "1 January 2099", + "startsAt": "2024-01-01T00:00:00.000Z", + "endsAt": "2099-01-01T00:00:00.000Z", } ], } diff --git a/src/threatzone/types/__init__.py b/src/threatzone/types/__init__.py index 77b41df..3bc9489 100644 --- a/src/threatzone/types/__init__.py +++ b/src/threatzone/types/__init__.py @@ -66,6 +66,7 @@ FileSource, Hashes, Message, + MetafieldValue, OperatingSystemPlatform, ReportOperatingSystem, ReportStatus, @@ -161,11 +162,12 @@ UserInfoDetails, WorkspaceBasicInfo, ) -from .mitre import MitreResponse, MitreTechniqueUsage +from .mitre import AttackDomain, MitreResponse, MitreTechniqueUsage from .network import ( Connection, ConnectionPackets, ConnectionProtocol, + ConnectOutcome, DnsQuery, DnsRecordType, DnsStatus, @@ -333,6 +335,7 @@ "ReportStatusValue", "ReportType", "Tag", + "MetafieldValue", "TagAuthor", "TagOrigin", "ThreatLevel", @@ -486,10 +489,12 @@ "KillchainPathAuthority", "KillchainProcess", # Mitre + "AttackDomain", "MitreResponse", "MitreTechniqueUsage", # Network "Connection", + "ConnectOutcome", "ConnectionPackets", "ConnectionProtocol", "DnsQuery", diff --git a/src/threatzone/types/common.py b/src/threatzone/types/common.py index c722739..cb404c7 100644 --- a/src/threatzone/types/common.py +++ b/src/threatzone/types/common.py @@ -27,6 +27,8 @@ | str ) TagAuthor = Literal["system", "user"] | str +MetafieldValue = bool | int | float | str +"""A stored analysis option value with its type.""" class Message(BaseModel): @@ -142,5 +144,14 @@ class ReportStatus(BaseModel): The key is the option name and the value is the selected parameter name. Only ``get_submission()`` sends it. ``None`` when the report has no options - and on older servers. + and on older servers. This is display text; read ``metafield_values`` in code. """ + metafield_values: dict[str, MetafieldValue] | None = Field( + default=None, alias="metafieldValues" + ) + """Stored analysis option values with their types, keyed by the metafield key, for + example ``{"timeout": 120, "internet_connection": True}``. Only ``get_submission()`` + sends it. ``None`` when the report has no options and on older servers.""" + execution_duration_seconds: float | None = Field(default=None, alias="executionDurationSeconds") + """The ``timeout`` option of a dynamic report in seconds. Only ``get_submission()`` + sends it. ``None`` for other reports and on older servers.""" diff --git a/src/threatzone/types/errors.py b/src/threatzone/types/errors.py index 28f1d0f..2910aa7 100644 --- a/src/threatzone/types/errors.py +++ b/src/threatzone/types/errors.py @@ -15,6 +15,8 @@ Literal[ "INVALID_QUERY_PARAM", "UNAUTHORIZED", + "WORKSPACE_INACTIVE", + "PLAN_API_ACCESS_DENIED", "LICENSE_INVALID", "FORBIDDEN", "NOT_FOUND", @@ -38,11 +40,10 @@ "QUOTA_UNAVAILABLE", "INTERNAL_ERROR", "STORAGE_UNAVAILABLE", + "BEHAVIOUR_INDEX_BUILDING", "BEHAVIOUR_INDEXING", "BEHAVIOUR_TOO_LARGE", "BEHAVIOUR_UNREADABLE", - "SAMPLE_INDEX_UNAVAILABLE", - "SAMPLE_INDEX_ERROR", "SESSION_SERVICE_UNAVAILABLE", "SESSION_NOT_FOUND", "AI_REPORT_NOT_FOUND", @@ -55,7 +56,8 @@ | str ) """The ``code`` values of the public API. Any other string is also accepted, so a code that -a newer server adds does not fail the parse.""" +a newer server adds does not fail the parse. ``BEHAVIOUR_INDEXING`` comes only from older +servers.""" class ApiError(BaseModel): diff --git a/src/threatzone/types/indicators.py b/src/threatzone/types/indicators.py index a832b7b..02631f1 100644 --- a/src/threatzone/types/indicators.py +++ b/src/threatzone/types/indicators.py @@ -79,6 +79,10 @@ class Indicator(BaseModel): model_config = ConfigDict(populate_by_name=True) id: str + source: str | None = None + """Engine of the row: ``dynamic_scan`` for every dynamic row, else the engine of an overview + row (for example ``static_scan`` or ``yara_scanner``). ``None`` when the row does not record + one and on older servers.""" name: str description: str summary: str | None = None @@ -88,8 +92,12 @@ class Indicator(BaseModel): level: IndicatorLevel score: float score_maximum: int | None = Field(default=None, alias="scoreMaximum") + """Deprecated: every score is on the 0-100 scale, so the value is always ``100``.""" pids: list[int] attack_codes: list[str] = Field(alias="attackCodes") + attempted_attack_codes: list[str] | None = Field(default=None, alias="attemptedAttackCodes") + """Techniques of ``attack_codes`` that the row marks attempted: the evidence is a request + that the sandbox did not confirm. ``None`` when the row has none and on older servers.""" event_ids: list[int] = Field(alias="eventIds") syscall_line_numbers: list[int] = Field(alias="syscallLineNumbers") author: IndicatorAuthor diff --git a/src/threatzone/types/me.py b/src/threatzone/types/me.py index 8447480..cfeae28 100644 --- a/src/threatzone/types/me.py +++ b/src/threatzone/types/me.py @@ -51,6 +51,9 @@ class FileLimits(BaseModel): extensions: list[str] file_size: str = Field(alias="fileSize") + """Display text, for example ``"3.00 GB"``. Read ``file_size_bytes`` in code.""" + file_size_bytes: int | None = Field(default=None, alias="fileSizeBytes") + """File size limit in bytes. ``None`` on older servers.""" class SubmissionLimits(BaseModel): @@ -91,7 +94,13 @@ class PlanInfo(BaseModel): plan_name: str = Field(alias="planName") start_time: str = Field(alias="startTime") + """Display text. Read ``starts_at`` in code.""" end_time: str = Field(alias="endTime") + """Display text. Read ``ends_at`` in code.""" + starts_at: datetime | None = Field(default=None, alias="startsAt") + """Start of the plan. ``None`` when there is no time and on older servers.""" + ends_at: datetime | None = Field(default=None, alias="endsAt") + """End of the plan. ``None`` when the plan does not end and on older servers.""" subs_time: str = Field(alias="subsTime") file_limits: FileLimits = Field(alias="fileLimits") submission_limits: SubmissionLimits = Field(alias="submissionLimits") @@ -106,7 +115,13 @@ class ModuleInfo(BaseModel): module_id: str = Field(alias="moduleId") module_name: str = Field(alias="moduleName") start_time: str = Field(alias="startTime") + """Display text. Read ``starts_at`` in code.""" end_time: str = Field(alias="endTime") + """Display text. Read ``ends_at`` in code.""" + starts_at: datetime | None = Field(default=None, alias="startsAt") + """Start of the module. ``None`` when there is no time and on older servers.""" + ends_at: datetime | None = Field(default=None, alias="endsAt") + """End of the module. ``None`` when the module does not end and on older servers.""" class UserInfo(BaseModel): diff --git a/src/threatzone/types/mitre.py b/src/threatzone/types/mitre.py index 96078f8..85ffdea 100644 --- a/src/threatzone/types/mitre.py +++ b/src/threatzone/types/mitre.py @@ -5,8 +5,14 @@ from __future__ import annotations +from typing import Literal + from pydantic import BaseModel, ConfigDict, Field +AttackDomain = Literal["enterprise", "mobile"] | str +"""ATT&CK domain of a report: ``mobile`` for an Android report, ``enterprise`` for the other +platforms.""" + class MitreTechniqueUsage(BaseModel): """One matched technique that is in the ATT&CK catalog of the platform.""" @@ -18,7 +24,7 @@ class MitreTechniqueUsage(BaseModel): name: str """Technique name from the ATT&CK catalog that ships with the platform.""" tactic_ids: list[str] = Field(alias="tacticIds") - """Tactic IDs (``TA####``) that contain the technique.""" + """Every tactic ID (``TA####``) of the technique, in ATT&CK matrix order.""" score: float """Highest score (0 to 100) of the indicators that carry the technique.""" score_maximum: int = Field(alias="scoreMaximum") @@ -26,6 +32,9 @@ class MitreTechniqueUsage(BaseModel): in_verdict_chain: bool = Field(alias="inVerdictChain") """``True`` when an indicator with this technique is part of the finding that set the verdict (Windows V2 reports). ``False`` for other reports.""" + attempted: bool | None = None + """``True`` when every indicator that carries the technique marks it attempted: the + evidence is a request that the sandbox did not confirm. ``None`` on older servers.""" indicator_ids: list[str] = Field(alias="indicatorIds") """IDs of the dynamic indicators that carry the technique (``Indicator.id``).""" @@ -37,6 +46,11 @@ class MitreResponse(BaseModel): techniques: list[str] total: int + attack_version: str | None = Field(default=None, alias="attackVersion") + """Version of the ATT&CK catalog, for example ``"20.0"``. ``None`` on older servers.""" + domain: AttackDomain | None = None + """ATT&CK domain of the report. ``items`` holds only the techniques of this domain. + ``None`` on older servers.""" items: list[MitreTechniqueUsage] | None = None """One entry for each matched technique that is in the ATT&CK catalog, sorted by ``technique_id``. A code that is not in the catalog is in ``techniques`` but not in diff --git a/src/threatzone/types/network.py b/src/threatzone/types/network.py index bb6babf..1da9ca0 100644 --- a/src/threatzone/types/network.py +++ b/src/threatzone/types/network.py @@ -32,27 +32,40 @@ "CAA", ] -DnsStatus = Literal[ - "NOERROR", - "FORMERR", - "SERVFAIL", - "NXDOMAIN", - "NOTIMP", - "REFUSED", - "YXDOMAIN", - "YXRRSET", - "NXRRSET", - "NOTAUTH", - "NOTZONE", - "BADVERS", - "BADKEY", - "BADTIME", - "BADMODE", - "BADNAME", - "BADALG", - "BADTRUNC", - "TIMEOUT", -] +DnsStatus = ( + Literal[ + "NOERROR", + "FORMERR", + "SERVFAIL", + "NXDOMAIN", + "NOTIMP", + "REFUSED", + "YXDOMAIN", + "YXRRSET", + "NXRRSET", + "NOTAUTH", + "NOTZONE", + "BADVERS", + "BADKEY", + "BADTIME", + "BADMODE", + "BADNAME", + "BADALG", + "BADTRUNC", + "TIMEOUT", + "UNKNOWN", + ] + | str +) +"""DNS response status. ``TIMEOUT`` means that the query got no answer. ``UNKNOWN`` means that +the sensor text is not a standard status; read ``status_text``.""" + +ConnectOutcome = Literal["established", "no_response", "refused", "not_captured"] | str +"""Outcome of a TCP or UDP connection attempt. + +``established``: the remote side answered. ``no_response``: nothing came back. ``refused``: the +remote side reset the connection. ``not_captured``: the process monitor saw the connect, but the +capture holds no flow for it, for example in a run without internet access.""" ConnectionProtocol = Literal["tcp", "udp"] ThreatProtocol = Literal["TCP", "UDP"] @@ -281,7 +294,12 @@ class DnsQuery(NetworkRowSource): status: DnsStatus status_text: str | None = Field(default=None, alias="statusText") records: list[str] + answered: bool | None = None + """``False`` when the query got no answer. ``None`` for rows of older reports and on older + servers.""" timeshift: float + timeshift_seconds: float | None = Field(default=None, alias="timeshiftSeconds") + """Seconds from the start of the capture. ``None`` when the server does not send it.""" source_scope: NetworkSourceScope | None = Field(default=None, alias="sourceScope") @@ -340,7 +358,12 @@ class Connection(NetworkRowSource): """Capture packets of the flow.""" packets_omitted: int | None = Field(default=None, alias="packetsOmitted") """Capture packets that the report packet limit left out. Download the PCAP to read them.""" + connect_outcome: ConnectOutcome | None = Field(default=None, alias="connectOutcome") + """Outcome of the connection attempt. ``None`` for rows of older reports and on older + servers.""" timeshift: float + timeshift_seconds: float | None = Field(default=None, alias="timeshiftSeconds") + """Seconds from the start of the capture. ``None`` when the server does not send it.""" source_scope: NetworkSourceScope | None = Field(default=None, alias="sourceScope") @@ -375,6 +398,8 @@ class NetworkThreat(NetworkRowSource): destination_ip: str = Field(alias="destinationIp") destination_port: int = Field(alias="destinationPort") timeshift: float + timeshift_seconds: float | None = Field(default=None, alias="timeshiftSeconds") + """Seconds from the start of the capture. ``None`` when the server does not send it.""" metadata: dict[str, list[str]] details: dict[str, str | int] tls: NetworkThreatTls | None = None @@ -525,7 +550,13 @@ class NetworkEndpoint(BaseModel): """DNS queries, HTTP requests and TCP and UDP flows of the endpoint.""" alert_count: int = Field(alias="alertCount") max_alert_severity: str | None = Field(default=None, alias="maxAlertSeverity") - """Highest alert severity as text. ``"1"`` is the highest. ``None`` without alerts.""" + """Deprecated: read ``max_alert_severity_level``. Highest alert severity as text. ``"1"`` is + the highest. ``None`` without alerts.""" + max_alert_severity_level: NetworkThreatSeverityLevel | None = Field( + default=None, alias="maxAlertSeverityLevel" + ) + """Highest alert severity, ``1`` (high) to ``4`` (info), on the scale of + ``NetworkThreat.severity_level``. ``None`` without alerts and on older servers.""" process_ids: list[int] = Field(alias="processIds") attribution_note: str | None = Field(default=None, alias="attributionNote") diff --git a/src/threatzone/types/submissions.py b/src/threatzone/types/submissions.py index d5f6b08..6379b63 100644 --- a/src/threatzone/types/submissions.py +++ b/src/threatzone/types/submissions.py @@ -103,7 +103,12 @@ class SubmissionListItem(BaseModel): uuid: str filename: str | None = None + """``None`` for a URL submission.""" sha256: str | None = None + """``None`` for a URL submission.""" + url: str | None = None + """Submitted URL of a URL submission. ``None`` for a file submission and on older + servers.""" level: SubmissionLevel type: SubmissionType private: bool diff --git a/src/threatzone/types/syscalls.py b/src/threatzone/types/syscalls.py index eebce6d..4df5c34 100644 --- a/src/threatzone/types/syscalls.py +++ b/src/threatzone/types/syscalls.py @@ -60,3 +60,6 @@ class SyscallSearchResult(BaseModel): busy: bool """``True`` when the server runs too many searches and did not scan. Send the same request again later.""" + retry_after_seconds: float | None = Field(default=None, alias="retryAfterSeconds") + """Seconds to wait before the same request when ``busy`` is ``True``. ``None`` on other + responses and on older servers.""" diff --git a/tests/integration/flows/test_flow_e2e_review_fields.py b/tests/integration/flows/test_flow_e2e_review_fields.py new file mode 100644 index 0000000..0a49c5c --- /dev/null +++ b/tests/integration/flows/test_flow_e2e_review_fields.py @@ -0,0 +1,55 @@ +"""Flow — the fake API returns the fields of the current Public API contract.""" + +from __future__ import annotations + +from threatzone import ThreatZone +from threatzone.testing import FakeThreatZoneAPI, scenarios + + +def _submit(fake_api: FakeThreatZoneAPI, sync_client: ThreatZone) -> str: + scenarios.seed_malicious_pe(fake_api) + created = sync_client.create_sandbox_submission(b"PE") + sync_client.wait_for_completion(created.uuid, poll_interval=0.001, timeout=5) + return created.uuid + + +def test_network_rows_carry_outcomes(fake_api: FakeThreatZoneAPI, sync_client: ThreatZone) -> None: + uuid = _submit(fake_api, sync_client) + assert {row.connect_outcome for row in sync_client.get_tcp_connections(uuid)} == {"established"} + assert all(row.answered for row in sync_client.get_dns_queries(uuid)) + + +def test_mitre_carries_the_catalog_version( + fake_api: FakeThreatZoneAPI, sync_client: ThreatZone +) -> None: + uuid = _submit(fake_api, sync_client) + mitre = sync_client.get_mitre_techniques(uuid) + assert (mitre.attack_version, mitre.domain) == ("20.0", "enterprise") + assert mitre.items and all(item.attempted is False for item in mitre.items) + + +def test_indicator_rows_carry_the_engine( + fake_api: FakeThreatZoneAPI, sync_client: ThreatZone +) -> None: + uuid = _submit(fake_api, sync_client) + assert {row.source for row in sync_client.get_indicators(uuid).items} == {"dynamic_scan"} + + +def test_url_list_item_has_url_and_no_file_fields( + fake_api: FakeThreatZoneAPI, sync_client: ThreatZone +) -> None: + scenarios.seed_phishing_url(fake_api, url="https://phishing.example.com/login") + created = sync_client.create_url_submission("https://phishing.example.com/login") + item = next(i for i in sync_client.list_submissions().items if i.uuid == created.uuid) + assert (item.url, item.filename, item.sha256) == ( + "https://phishing.example.com/login", + None, + None, + ) + + +def test_me_carries_machine_readable_plan_fields(sync_client: ThreatZone) -> None: + me = sync_client.get_user_info() + assert me.plan.starts_at is not None + assert me.plan.file_limits.file_size_bytes == 104857600 + assert me.modules[0].ends_at is not None diff --git a/tests/test_api_e2e_review_shapes.py b/tests/test_api_e2e_review_shapes.py new file mode 100644 index 0000000..4909279 --- /dev/null +++ b/tests/test_api_e2e_review_shapes.py @@ -0,0 +1,695 @@ +"""Payloads shaped like current Public API responses (field names, nulls and value types).""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any + +import httpx +import pytest + +from threatzone import ( + BadRequestError, + BehaviourIndexBuildingError, + BehaviourIndexingError, + InternalServerError, + PaymentRequiredError, + PermissionDeniedError, + QuotaUnavailableError, + ServiceUnavailableError, +) +from threatzone._exceptions import raise_for_status +from threatzone.types import ( + Artifact, + BehaviourEvent, + Connection, + DnsQuery, + Indicator, + MitreResponse, + NetworkEndpoint, + NetworkThreat, + PaginatedSubmissions, + Submission, + SyscallSearchResult, + UserInfo, +) +from threatzone.types.errors import ApiError + +UUID_A = "00000000-0000-4000-8000-000000000001" +UUID_URL = "00000000-0000-4000-8000-000000000002" +SHA256 = "a" * 64 + + +def _row_source() -> dict[str, Any]: + return { + "sourceScope": "submission_ip_scoped", + "attribution": "sensor_only", + "attributionReason": None, + "processId": None, + "processImage": None, + } + + +def _tcp_row(**overrides: Any) -> dict[str, Any]: + row: dict[str, Any] = { + "id": "65f0aa000000000000000101", + "protocol": "tcp", + "sourceIp": "192.0.2.10", + "sourcePort": 49676, + "destinationIp": "203.0.113.7", + "destinationPort": 80, + "domain": "cdn.example.com", + "ptrDomain": "cdn.example.com", + "asn": "EXAMPLE-ASN", + "country": "US", + "packets": {"sent": 0, "received": 1, "empty": False}, + "appProto": None, + "hostname": None, + "pktsToServer": None, + "pktsToClient": None, + "bytesToServer": None, + "bytesToClient": None, + "packetsTotal": None, + "packetsOmitted": None, + "connectOutcome": None, + "timeshift": 0, + **_row_source(), + } + row.update(overrides) + return row + + +def _dns_row(**overrides: Any) -> dict[str, Any]: + row: dict[str, Any] = { + "id": "65f0aa000000000000000102", + "host": "api.example.com", + "type": "CNAME", + "status": "NOERROR", + "statusText": "NOERROR", + "records": ["edge.example.net"], + "answered": None, + "timeshift": 1660, + **_row_source(), + } + row.update(overrides) + return row + + +class TestNetworkRowFields: + @pytest.mark.parametrize( + "outcome", ["established", "no_response", "refused", "not_captured", None] + ) + def test_connection_connect_outcome(self, outcome: str | None) -> None: + row = Connection.model_validate(_tcp_row(connectOutcome=outcome)) + assert row.connect_outcome == outcome + + def test_connection_accepts_future_connect_outcome(self) -> None: + row = Connection.model_validate(_tcp_row(connectOutcome="reset_late")) + assert row.connect_outcome == "reset_late" + + def test_connection_without_new_fields_from_older_server(self) -> None: + payload = _tcp_row() + del payload["connectOutcome"] + row = Connection.model_validate(payload) + assert row.connect_outcome is None + assert row.timeshift_seconds is None + + def test_timeshift_seconds_on_rows(self) -> None: + assert Connection.model_validate(_tcp_row(timeshiftSeconds=1.66)).timeshift_seconds == 1.66 + assert DnsQuery.model_validate(_dns_row(timeshiftSeconds=1.66)).timeshift_seconds == 1.66 + + @pytest.mark.parametrize("answered", [True, False, None]) + def test_dns_answered(self, answered: bool | None) -> None: + assert DnsQuery.model_validate(_dns_row(answered=answered)).answered is answered + + def test_dns_unknown_status(self) -> None: + row = DnsQuery.model_validate(_dns_row(status="UNKNOWN", statusText="weird sensor text")) + assert row.status == "UNKNOWN" + + def test_dns_timeout_status_without_answer(self) -> None: + row = DnsQuery.model_validate(_dns_row(status="TIMEOUT", answered=False, records=[])) + assert (row.status, row.answered) == ("TIMEOUT", False) + + def test_threat_info_severity(self) -> None: + threat = NetworkThreat.model_validate( + { + "signature": "ET INFO Example Lookup", + "description": "Example lookup", + "severity": "info", + "severityLevel": 4, + "protocol": "UDP", + "appProto": "DNS", + "sourceIp": "192.0.2.10", + "sourcePort": 53000, + "destinationIp": "198.51.100.53", + "destinationPort": 53, + "timeshift": 2.5, + "metadata": {}, + "details": {}, + **_row_source(), + } + ) + assert (threat.severity, threat.severity_level) == ("info", 4) + + def test_endpoint_max_alert_severity_level(self) -> None: + endpoint = NetworkEndpoint.model_validate( + { + "key": "c2.example.com", + "host": "c2.example.com", + "ips": ["203.0.113.9"], + "ports": [4212], + "protocols": ["DNS", "TCP"], + "source": "sample", + "firstTimeshift": 16600, + "lastTimeshift": 37770, + "requestCount": 2, + "alertCount": 1, + "maxAlertSeverity": "1", + "processIds": [576], + "attributionNote": None, + "maxAlertSeverityLevel": 1, + } + ) + assert endpoint.max_alert_severity_level == 1 + assert endpoint.max_alert_severity == "1" + + +class TestMitreV20Fields: + def test_mitre_response(self) -> None: + mitre = MitreResponse.model_validate( + { + "techniques": ["T1055", "T1685"], + "total": 2, + "attackVersion": "20.0", + "domain": "enterprise", + "items": [ + { + "techniqueId": "T1055", + "name": "Process Injection", + "tacticIds": ["TA0004", "TA0005"], + "score": 70, + "scoreMaximum": 100, + "inVerdictChain": True, + "attempted": False, + "indicatorIds": ["65f0aa000000000000000201"], + }, + { + "techniqueId": "T1685", + "name": "Disable or Modify Tools", + "tacticIds": ["TA0005"], + "score": 45, + "scoreMaximum": 100, + "inVerdictChain": False, + "attempted": True, + "indicatorIds": ["65f0aa000000000000000202"], + }, + ], + } + ) + assert (mitre.attack_version, mitre.domain) == ("20.0", "enterprise") + assert mitre.items is not None + assert mitre.items[0].tactic_ids == ["TA0004", "TA0005"] + assert [item.attempted for item in mitre.items] == [False, True] + + def test_mitre_response_from_older_server(self) -> None: + mitre = MitreResponse.model_validate({"techniques": ["T1055"], "total": 1}) + assert (mitre.attack_version, mitre.domain) == (None, None) + + def test_mobile_domain(self) -> None: + mitre = MitreResponse.model_validate( + {"techniques": [], "total": 0, "attackVersion": "20.0", "domain": "mobile", "items": []} + ) + assert mitre.domain == "mobile" + + +def _indicator(**overrides: Any) -> dict[str, Any]: + row: dict[str, Any] = { + "id": "65f0aa000000000000000201", + "source": "dynamic_scan", + "name": "Code injection into a started process", + "description": "The sample placed code in a process that it started.", + "summary": "The sample placed code in a process that it started.", + "category": ["injection"], + "level": "malicious", + "score": 70, + "pids": [576], + "attackCodes": ["T1055"], + "eventIds": [5458, 5472], + "syscallLineNumbers": [5459, 5473], + "author": "system", + "scoreMaximum": 100, + } + row.update(overrides) + return row + + +class TestIndicatorFields: + @pytest.mark.parametrize("source", ["dynamic_scan", "yara_scanner", "config_extractor", None]) + def test_source(self, source: str | None) -> None: + assert Indicator.model_validate(_indicator(source=source)).source == source + + def test_attempted_attack_codes(self) -> None: + indicator = Indicator.model_validate( + _indicator(attackCodes=["T1055", "T1685"], attemptedAttackCodes=["T1685"]) + ) + assert indicator.attempted_attack_codes == ["T1685"] + + def test_attempted_attack_codes_absent(self) -> None: + assert Indicator.model_validate(_indicator()).attempted_attack_codes is None + + +def _list_item(**overrides: Any) -> dict[str, Any]: + item: dict[str, Any] = { + "uuid": UUID_A, + "filename": "sample.exe", + "sha256": SHA256, + "url": None, + "level": "malicious", + "type": "file", + "private": False, + "tags": [{"author": "system", "type": "malicious", "value": "rat", "origin": "dynamic"}], + "reports": [ + { + "type": "dynamic", + "status": "completed", + "level": "malicious", + "duration": 318.4, + "format": "regular", + "operatingSystem": {"name": "Windows 10 x64", "platform": "windows"}, + "score": 70, + } + ], + "mimetype": "application/x-msdownload", + "size": 257536, + "overview": { + "status": "completed", + "jobs": { + "total": 5, + "succeeded": 5, + "failed": 0, + "skipped": 0, + "finished": 5, + "completed": 5, + }, + }, + "indicators": { + "levels": {"malicious": 3, "suspicious": 2, "benign": 8}, + "artifactCount": 1, + }, + "createdAt": "2026-10-19T16:43:27.691Z", + } + item.update(overrides) + return item + + +def _url_list_item() -> dict[str, Any]: + return _list_item( + uuid=UUID_URL, + filename=None, + sha256=None, + url="https://www.example.org/", + level="benign", + type="url", + tags=[], + reports=[ + { + "type": "url_analysis", + "status": "completed", + "level": "benign", + "duration": None, + "format": "regular", + "operatingSystem": None, + "score": 0, + } + ], + mimetype=None, + size=None, + overview={"status": "completed", "jobs": None}, + indicators={"levels": {"malicious": 0, "suspicious": 0, "benign": 0}, "artifactCount": 0}, + ) + + +def _detail(**overrides: Any) -> dict[str, Any]: + detail: dict[str, Any] = { + "uuid": UUID_A, + "type": "file", + "level": "malicious", + "score": 70, + "family": None, + "private": False, + "tags": [], + "reports": [ + { + "type": "dynamic", + "status": "completed", + "level": "malicious", + "duration": 280.1, + "format": "regular", + "operatingSystem": {"name": "Windows 10 x64", "platform": "windows"}, + "score": 70, + "metafields": {"Execution Duration": "120 Seconds", "Work Path": "Desktop"}, + "metafieldValues": { + "private": False, + "timeout": 120, + "work_path": "desktop", + "trace_profile": "65f0aa000000000000000301", + }, + "executionDurationSeconds": 120, + }, + { + "type": "static", + "status": "completed", + "level": "malicious", + "duration": 9.8, + "format": "exe", + "operatingSystem": None, + "score": 100, + "metafields": None, + "metafieldValues": None, + "executionDurationSeconds": None, + }, + ], + "overview": {"status": "completed", "jobs": None, "duration": 280.1}, + "indicators": { + "levels": {"malicious": 6, "suspicious": 2, "benign": 11}, + "artifactCount": 1, + }, + "mitreTechniques": ["T1055"], + "createdAt": "2026-10-19T13:58:44.959Z", + "updatedAt": "2026-10-19T13:59:48.038Z", + "scoreBreakdown": None, + } + detail.update(overrides) + return detail + + +class TestSubmissionFields: + def test_list_url_and_null_file_fields(self) -> None: + page = PaginatedSubmissions.model_validate( + { + "items": [_list_item(), _url_list_item()], + "total": 2, + "page": 1, + "limit": 50, + "totalPages": 1, + } + ) + file_item, url_item = page.items + assert (file_item.url, file_item.filename, file_item.sha256) == ( + None, + "sample.exe", + SHA256, + ) + assert (url_item.url, url_item.filename, url_item.sha256) == ( + "https://www.example.org/", + None, + None, + ) + + def test_report_metafield_values(self) -> None: + submission = Submission.model_validate(_detail()) + dynamic, static = submission.reports + assert dynamic.metafield_values == { + "private": False, + "timeout": 120, + "work_path": "desktop", + "trace_profile": "65f0aa000000000000000301", + } + assert isinstance(dynamic.metafield_values["timeout"], int) + assert isinstance(dynamic.metafield_values["private"], bool) + assert dynamic.execution_duration_seconds == 120 + assert (static.metafield_values, static.execution_duration_seconds) == (None, None) + + def test_url_detail_without_file_fields(self) -> None: + submission = Submission.model_validate( + _detail( + uuid=UUID_URL, + type="url", + url="https://www.example.org/", + reports=[ + { + "type": "url_analysis", + "status": "completed", + "level": "benign", + "duration": None, + "format": "regular", + "operatingSystem": None, + "score": 0, + "metafields": {"Execution Duration": "180 Seconds"}, + "metafieldValues": {"timeout": 180, "record_session": True}, + "executionDurationSeconds": None, + } + ], + ) + ) + assert (submission.filename, submission.hashes, submission.file) == (None, None, None) + assert submission.reports[0].metafield_values == {"timeout": 180, "record_session": True} + + +def _me(**plan_overrides: Any) -> dict[str, Any]: + plan: dict[str, Any] = { + "planName": "Researcher", + "startTime": "15 October 2026", + "endTime": "Unlimited", + "startsAt": "2026-10-15T10:55:26.100Z", + "endsAt": None, + "subsTime": "infinite", + "fileLimits": {"extensions": [], "fileSize": "3.00 GB", "fileSizeBytes": 3221225472}, + "submissionLimits": { + "apiLimit": 1000000, + "dailyLimit": 10000, + "concurrentLimit": 100, + "aiReportLimit": 20, + }, + "permissions": {"downloadSample": True, "aiReport": True}, + } + plan.update(plan_overrides) + return { + "userInfo": { + "email": "analyst@example.com", + "fullName": "Analyst", + "workspace": { + "name": "Example Workspace", + "alias": "example-workspace", + "private": False, + "type": "personal", + }, + "limitsCount": { + "apiRequestCount": 454, + "dailySubmissionCount": 9, + "concurrentSubmissionCount": 0, + "aiReportCount": 1, + "aiReportResetsAt": "2026-10-20T00:00:00.000Z", + }, + }, + "plan": plan, + "modules": [ + { + "moduleId": "65f0aa000000000000000401", + "moduleName": "CDR", + "startTime": "1 October 2026", + "endTime": "1 October 2027", + "startsAt": "2026-10-01T00:00:00.000Z", + "endsAt": "2027-10-01T00:00:00.000Z", + } + ], + } + + +class TestMeFields: + def test_machine_readable_plan_fields(self) -> None: + me = UserInfo.model_validate(_me()) + assert me.plan.starts_at == datetime(2026, 10, 15, 10, 55, 26, 100000, tzinfo=timezone.utc) + assert me.plan.ends_at is None + assert me.plan.file_limits.file_size_bytes == 3221225472 + assert me.modules[0].starts_at == datetime(2026, 10, 1, tzinfo=timezone.utc) + assert me.modules[0].ends_at == datetime(2027, 10, 1, tzinfo=timezone.utc) + + def test_older_server_without_machine_fields(self) -> None: + payload = _me() + for key in ("startsAt", "endsAt"): + del payload["plan"][key] + del payload["modules"][0][key] + del payload["plan"]["fileLimits"]["fileSizeBytes"] + me = UserInfo.model_validate(payload) + assert (me.plan.starts_at, me.plan.file_limits.file_size_bytes) == (None, None) + + +class TestRegressionShapes: + def test_behaviour_event_fractional_timestamp(self) -> None: + event = BehaviourEvent.model_validate( + { + "type": "process", + "pid": 576, + "ppid": 4, + "operation": "create", + "eventId": 1, + "eventCount": 1, + "timestamp": 1790376279.55, + "details": {}, + } + ) + assert event.timestamp == 1790376279.55 + + @pytest.mark.parametrize("artifact_type", ["email_file", "email_attachment"]) + def test_email_artifact_types(self, artifact_type: str) -> None: + artifact = Artifact.model_validate( + { + "id": "65f0aa000000000000000501", + "filename": "message.eml", + "size": 2048, + "type": artifact_type, + "source": "static_report", + "hashes": {"md5": "b" * 32, "sha1": "c" * 40, "sha256": "d" * 64}, + "tags": [], + } + ) + assert artifact.type == artifact_type + + def test_syscall_search_busy_retry_after_seconds(self) -> None: + result = SyscallSearchResult.model_validate( + { + "lines": [], + "nextFromLine": 0, + "scannedTo": 0, + "totalCount": None, + "done": False, + "truncated": False, + "busy": True, + "retryAfterSeconds": 5, + } + ) + assert result.retry_after_seconds == 5 + + +def _error(status: int, body: dict[str, Any], headers: dict[str, str] | None = None) -> None: + raise_for_status( + httpx.Response( + status, + json=body, + headers=headers, + request=httpx.Request("GET", "https://test.threat.zone/x"), + ) + ) + + +class TestErrorCodes: + def test_workspace_inactive(self) -> None: + with pytest.raises(PaymentRequiredError) as exc_info: + _error( + 402, + { + "statusCode": 402, + "error": "Payment Required", + "message": "The workspace is not active. Renew the plan.", + "code": "WORKSPACE_INACTIVE", + }, + ) + assert exc_info.value.code == "WORKSPACE_INACTIVE" + + def test_plan_api_access_denied(self) -> None: + with pytest.raises(PermissionDeniedError) as exc_info: + _error( + 403, + { + "statusCode": 403, + "error": "Forbidden", + "message": "The plan of the workspace does not include API access.", + "code": "PLAN_API_ACCESS_DENIED", + }, + ) + assert exc_info.value.code == "PLAN_API_ACCESS_DENIED" + + def test_code_is_none_without_body(self) -> None: + with pytest.raises(InternalServerError) as exc_info: + raise_for_status( + httpx.Response( + 502, + content=b"Bad Gateway", + request=httpx.Request("GET", "https://test.threat.zone/x"), + ) + ) + assert exc_info.value.code is None + + def test_validation_field(self) -> None: + with pytest.raises(BadRequestError) as exc_info: + _error( + 400, + { + "statusCode": 400, + "error": "Bad Request", + "message": "limit must not be greater than 100", + "code": "INVALID_QUERY_PARAM", + "details": {"field": "limit"}, + }, + ) + assert (exc_info.value.code, exc_info.value.field) == ("INVALID_QUERY_PARAM", "limit") + + def test_validation_without_field(self) -> None: + with pytest.raises(BadRequestError) as exc_info: + _error(400, {"statusCode": 400, "message": "Bad", "code": "INVALID_QUERY_PARAM"}) + assert exc_info.value.field is None + + def test_behaviour_index_building(self) -> None: + with pytest.raises(BehaviourIndexBuildingError) as exc_info: + _error( + 503, + { + "statusCode": 503, + "error": "Service Unavailable", + "message": "The behaviour index is being built.", + "code": "BEHAVIOUR_INDEX_BUILDING", + "details": {"retryAfterSeconds": 30}, + }, + headers={"Retry-After": "30"}, + ) + error = exc_info.value + assert isinstance(error, BehaviourIndexingError) + assert isinstance(error, ServiceUnavailableError) + assert isinstance(error, InternalServerError) + assert (error.status_code, error.code, error.retry_after) == ( + 503, + "BEHAVIOUR_INDEX_BUILDING", + 30.0, + ) + + def test_service_unavailable_retry_after_from_details(self) -> None: + with pytest.raises(ServiceUnavailableError) as exc_info: + _error( + 503, + { + "statusCode": 503, + "message": "Storage is not available.", + "code": "STORAGE_UNAVAILABLE", + "details": {"retryAfterSeconds": 15}, + }, + ) + assert (exc_info.value.code, exc_info.value.retry_after) == ("STORAGE_UNAVAILABLE", 15.0) + + def test_quota_unavailable_is_service_unavailable(self) -> None: + with pytest.raises(QuotaUnavailableError) as exc_info: + _error( + 503, + { + "statusCode": 503, + "message": "The quota cannot be checked.", + "code": "QUOTA_UNAVAILABLE", + "retryAfter": 10, + "details": {"retryAfterSeconds": 10}, + }, + headers={"Retry-After": "10"}, + ) + assert isinstance(exc_info.value, ServiceUnavailableError) + assert exc_info.value.retry_after == 10.0 + + def test_api_error_envelope_with_field(self) -> None: + error = ApiError.model_validate( + { + "statusCode": 400, + "error": "Bad Request", + "message": "limit must not be greater than 100", + "code": "INVALID_QUERY_PARAM", + "details": {"field": "limit"}, + } + ) + assert error.details == {"field": "limit"}