From c71d9d4da3593ae5c5f20e397735eecc395dad32 Mon Sep 17 00:00:00 2001 From: Dmitry Ermakovich Date: Fri, 28 Aug 2026 10:47:50 +0300 Subject: [PATCH 1/3] fix: issue with testResult steps and run finish --- .gitignore | 3 +- README.md | 4 + docs/README.md | 17 ++ ...0-duplicate-results-bulk-after-realtime.md | 62 ++++ docs/mode0-orphan-testpoint-inprogress.md | 90 ++---- docs/test-result-export-contract.md | 117 +++++++ testit-adapter-behave/setup.py | 2 +- testit-adapter-nose/setup.py | 2 +- .../docs/displayName-flow.md | 285 ++++++++++++++++++ testit-adapter-pytest/setup.py | 2 +- testit-adapter-robotframework/setup.py | 2 +- testit-python-commons/README.md | 4 +- testit-python-commons/realtime-import-spec.md | 129 +++----- testit-python-commons/setup.py | 2 +- .../client/api_client.py | 121 +++----- .../services/adapter_manager.py | 11 +- .../tests/client/test_find_in_progress.py | 135 --------- .../test_update_existing_test_result.py | 69 +++++ .../client/test_write_tests_bulk_skip.py | 83 +++++ .../tests/services/test_adapter_manager.py | 1 + update_versions.sh | 2 +- 21 files changed, 769 insertions(+), 374 deletions(-) create mode 100644 docs/README.md create mode 100644 docs/mode0-duplicate-results-bulk-after-realtime.md create mode 100644 docs/test-result-export-contract.md create mode 100644 testit-adapter-pytest/docs/displayName-flow.md create mode 100644 testit-python-commons/tests/client/test_update_existing_test_result.py create mode 100644 testit-python-commons/tests/client/test_write_tests_bulk_skip.py diff --git a/.gitignore b/.gitignore index 11de2a6b..38aad347 100644 --- a/.gitignore +++ b/.gitignore @@ -59,4 +59,5 @@ pip-wheel-metadata/ pytestdebug.log .vs/ -*-examples/ \ No newline at end of file +*-examples/ +.user/ \ No newline at end of file diff --git a/README.md b/README.md index 652468b9..ba917362 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,10 @@ Supported test frameworks: 3. [RobotFramework](https://github.com/testit-tms/adapters-python/tree/main/testit-adapter-robotframework) 4. [Nose](https://github.com/testit-tms/adapters-python/tree/main/testit-adapter-nose) +## Internal documentation + +Adapter behaviour specs (export contract, mode 0, real-time import): [docs/README.md](docs/README.md). + # 🚀 Warning Since 3.0.0 version: - If the externalId annotation is not specified, then its contents will be a hash of a fully qualified method name. diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..b8c86973 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,17 @@ +# Adapter documentation (Python) + +Supplementary docs for behaviour that is not covered in adapter READMEs. + +| Document | Topic | +|----------|--------| +| [test-result-export-contract.md](./test-result-export-contract.md) | **Canonical:** `sendTestResults` vs PUT; PUT only for fixture setup/teardown | +| [mode0-duplicate-results-bulk-after-realtime.md](./mode0-duplicate-results-bulk-after-realtime.md) | Mode 0 duplicate Passed (TP + orphan): no second bulk send | +| [mode0-orphan-testpoint-inprogress.md](./mode0-orphan-testpoint-inprogress.md) | Mode 0 + InProgress slots: create path, Sync Storage | +| [test-run-tags-and-links.md](./test-run-tags-and-links.md) | Test run tags & links on create / early merge | + +Commons (real-time import, nested steps): + +| Document | Topic | +|----------|--------| +| [../testit-python-commons/realtime-import-spec.md](../testit-python-commons/realtime-import-spec.md) | `importRealtime=true`, nested steps, session PUT | +| [../testit-python-commons/sync-storage-interaction-spec.md](../testit-python-commons/sync-storage-interaction-spec.md) | Sync Storage protocol | diff --git a/docs/mode0-duplicate-results-bulk-after-realtime.md b/docs/mode0-duplicate-results-bulk-after-realtime.md new file mode 100644 index 00000000..0a426903 --- /dev/null +++ b/docs/mode0-duplicate-results-bulk-after-realtime.md @@ -0,0 +1,62 @@ +# Mode 0: duplicate test results (TP-bound + orphan) — Python adapter + +**Related:** [mode0-orphan-testpoint-inprogress.md](./mode0-orphan-testpoint-inprogress.md), [test-result-export-contract.md](./test-result-export-contract.md) +**Typical setup:** pytest + Sync Storage, `adapterMode=0`, fixed `testRunId`, `importRealtime=false` (default) + +Cross-adapter parity: Java [PR #278](https://github.com/testit-tms/adapters-java/pull/278). + +--- + +## Symptom (historical bug) + +One autotest from a test plan produced **two** Passed rows in the same run for the same `externalId`: + +| Row | `testPointId` | Origin | +|-----|---------------|--------| +| Older | Real plan point UUID | Created by TMS when the run started | +| Newer | Missing / `00000000-…` | **Second** `POST …/testRuns/{id}/test-results` from the adapter | + +Root cause: the adapter finalized the test twice — once at **test finish**, again at **run finish** (bulk). + +--- + +## Current behaviour (fixed) + +**Rule:** final status goes only through **`sendTestResults`**. +**PUT** is **not** used to finalize; see [test-result-export-contract.md](./test-result-export-contract.md). + +### End of test — Sync Storage master (`importRealtime=false`) + +When Sync Storage accepts the cut (`on_master_no_already_in_progress` → true): + +1. Cut model to Sync Storage (coordination only). +2. `_write_test_realtime_internal` → autotest update + **`sendTestResults`** with final status and full steps. +3. Store `externalId → resultId` in `AdapterManager.__test_result_map`. + +### End of run — bulk (`write_tests_after_all`) + +- If `externalId` is already in `__test_result_map` → **skip** `sendTestResults`; refresh autotest metadata if needed. +- Otherwise → bulk `sendTestResults` once (Sync Storage off, or test not sent at finish). + +--- + +## Flow (mode 0 + Sync Storage + `importRealtime=false`) + +```text +Plan start → TMS creates TP-bound InProgress +stopTestCase → SyncStorage cut + sendTestResults (Passed/Failed + full payload) +sessionfinish → bulk skips sendTestResults for tests already in __test_result_map +``` + +Expected: **one** finalized result row per autotest in the run, with steps from the create payload. + +--- + +## Regression checklist + +1. Mode 0, one autotest, Sync Storage on, `importRealtime=false`. +2. Log: `Finalized test result via sendTestResults` at test end. +3. Log: `Bulk import: skip sendTestResults …` at run end. +4. `testResults/search`: one hit per `externalId`. + +**Bad signs:** two Passed rows for the same `externalId`; bulk `sendTestResults` without skip for an already finalized test. diff --git a/docs/mode0-orphan-testpoint-inprogress.md b/docs/mode0-orphan-testpoint-inprogress.md index 2d2c9b28..c9d225a1 100644 --- a/docs/mode0-orphan-testpoint-inprogress.md +++ b/docs/mode0-orphan-testpoint-inprogress.md @@ -1,89 +1,61 @@ -# Mode 0: InProgress matching (orphan TP + parametrized) +# Mode 0: InProgress slots and test plan runs -How the Python adapter finds and **updates** an existing InProgress test result -instead of creating a new one when `adapterMode=0` (test plan / webhook / filter). +How the Python adapter exports results when `adapterMode=0` (test plan / webhook / filter) and TMS already created **InProgress** rows bound to test points. -Related: Java/sync-storage contract in -`adapters-java/docs/mode0-orphan-testpoint-inprogress.md`. +**Related:** [test-result-export-contract.md](./test-result-export-contract.md), Java `adapters-java/docs/mode0-orphan-testpoint-inprogress.md`. --- -## Problem 1 — orphan InProgress (no `testPointId`) +## TMS context -With `adapterMode=0`, TMS already creates InProgress results **bound to a test point** -(`testPointId` set). +With `adapterMode=0`, the test run is created from a plan. TMS pre-creates one **InProgress** result per test point (`testPointId` set). -Previously the Python adapter (after sync-storage accepted the cut) forced -`outcome=InProgress` and called create (`setAutoTestResultsForTestRun`) **without** -`testPointId`. That produced a second row (orphan). The TP-bound result stayed -InProgress forever; the final status landed on the orphan. - -**Fix:** keep the final outcome after sync; before create, search InProgress by -`autotest_external_id`, prefer a result with a valid `testPointId`, then **PUT** -that result instead of posting a new one. +The adapter must **finalize** those rows with the real outcome and full payload (steps, parameters, …), not leave them stuck InProgress and not create orphan duplicates. --- -## Problem 2 — parametrized pytest leaves many InProgress forever - -### What happens in practice +## Correct behaviour (current) -1. A test plan / filter creates many InProgress rows (one per test point). -2. The same autotest is parametrized (`@pytest.mark.parametrize`). Iterations often - share one `external_id` (name template without `{param}`). -3. Work items / test points frequently have **empty parameters**, while the adapter - sends **callspec parameters** on create. -4. The first matching logic used only `external_id`. The first TP-bound InProgress - was always chosen (or create ran when params on TMS did not “fit” create semantics). -5. Result: one empty TP might get updated once; other iterations **POST new results** - with parameters; remaining empty InProgress rows stay InProgress forever. +**Finalization uses `sendTestResults` only** (`POST /adapters/testRuns/{id}/test-results`). -Root cause: mismatch between WI parameters and -autotest/result parameters, plus search that ignores parameters. +- Applies even when an InProgress result already exists for the autotest. +- TMS merges the create payload into the plan-bound row (external id + configuration + parameters). +- The adapter **does not** search InProgress and **does not** PUT final status (removed workaround). -### Why short search is not enough +After Sync Storage accepts the cut, the adapter keeps the **final** outcome and calls `_write_test_realtime_internal` → `sendTestResults`. -`TestResultShortResponse` (search) has **no `parameters`** / `testPointId`. -Full detail comes from `GET /adapters/testResults/{id}` (`TestResultResponse`). +See [test-result-export-contract.md](./test-result-export-contract.md) and [mode0-duplicate-results-bulk-after-realtime.md](./mode0-duplicate-results-bulk-after-realtime.md). --- -## Solution (parameter-aware pick + claim) +## Historical issues (for context) + +### Orphan without `testPointId` + +Older behaviour: POST create without matching the plan row → second result (orphan), plan row stayed InProgress. -| File | Role | -|------|------| -| `client/helpers/test_result_matching.py` | Pure ranking: exact params > empty TMS params; prefer valid TP | -| `client/api_client.py` | `find_in_progress_test_result_id(external_id, parameters)`; claim chosen ids; `__load_test_result` passes `get_parameters()` | -| `services/adapter_manager.py` | After sync: keep **final** outcome; realtime write without forcing InProgress | +**Fix:** finalize via `sendTestResults`; TMS updates the existing slot instead of requiring a separate PUT-by-id path. -### Matching rules +### Duplicate at bulk (importRealtime=false) -For candidates with the same `external_id` (and not yet claimed in this process): +Test finalized at test finish via `sendTestResults`, then bulk at session end sent **again** → orphan duplicate. -1. **Exact parameter match** (normalized string key/value) — best. -2. Else **empty / missing parameters on TMS** — fallback for WI without params - (typical test-plan case). -3. Else **skip** if TMS has non-empty parameters that differ from the incoming ones - (do not overwrite the wrong callspec’s TP). -4. Among equal match quality, prefer a valid `testPointId`. -5. **Claim** the chosen result id so the next parametrize iteration does not reuse - the same empty TP. +**Fix:** skip bulk `sendTestResults` when `externalId` is already in `AdapterManager.__test_result_map`. -If nothing matches → create (previous behaviour for genuinely new results). +### Parametrized tests and many InProgress rows -Non-parametrized tests (`parameters` empty/`None`) still pick the empty-params -TP-bound InProgress first — same as the original orphan fix. +When several TPs share the same `external_id` with different (or empty) parameters, TMS matching on create uses the payload `parameters`. Adapter sends callspec parameters on `sendTestResults`; each iteration should get its own finalized row without leaving unrelated TPs stuck (TMS-side matching). + +Helper `client/helpers/test_result_matching.py` remains for unit tests / future client-side ranking if needed; **`__load_test_result` no longer uses it**. + +--- -### What we do **not** change +## What PUT is still used for -- Adapters PUT body still cannot set `parameters` (OpenAPI PUT has no field) — - we complete the existing TP-bound row (outcome / duration / message / trace). -- Cloud-side “better” filling of WI params remains a TMS concern; the adapter only - stops creating endless orphans and stuck InProgress. +**Only** after session with `importRealtime=true`: attach pytest fixture **setup/teardown** steps to an already finalized result id. No status, no `stepResults`. See [test-result-export-contract.md](./test-result-export-contract.md). --- ## Note -Sync-storage Work X PUT fix still requires a published binary newer than -`v0.3.7-tms-5.7` for nested-step finalize issues unrelated to this matching. +Sync Storage Work X finalize may still affect nested steps on the **first** held test independently of this export contract; see [realtime-import-spec.md](../testit-python-commons/realtime-import-spec.md) (Cause B). diff --git a/docs/test-result-export-contract.md b/docs/test-result-export-contract.md new file mode 100644 index 00000000..8c2428bd --- /dev/null +++ b/docs/test-result-export-contract.md @@ -0,0 +1,117 @@ +# Test result export contract (Python adapter) + +Canonical rules for how the Python adapter sends test results to TMS. +Aligned with Java adapters [PR #278](https://github.com/testit-tms/adapters-java/pull/278) (TMS-41083). + +**Related:** [realtime-import-spec.md](../testit-python-commons/realtime-import-spec.md), [mode0-duplicate-results-bulk-after-realtime.md](./mode0-duplicate-results-bulk-after-realtime.md) + +--- + +## API methods + +| TMS operation | Adapter API | When used | +|---------------|-------------|-----------| +| **Send test results** | `POST /adapters/testRuns/{id}/test-results` (`adapters_test_runs_id_test_results_post`, Java: `sendTestResults`) | **Final** test result: status, steps, parameters, message, attachments | +| **Update test result** | `PUT /adapters/testResults/{id}` (`adapters_test_results_id_put`) | **Only** fixture `setupResults` / `teardownResults` after session (see below) | + +--- + +## Rules (correct behaviour) + +### 1. Finalization = `sendTestResults` only + +- Every finalized test result (Passed, Failed, Skipped, …) is sent via **`sendTestResults`**, including when TMS already has an **InProgress** row from a test plan (`adapterMode=0` / `1`). +- TMS merges/enriches the existing InProgress slot on create; the adapter **must not** finalize via PUT. +- Full nested `stepResults` use `AttachmentPutModelAutoTestStepResultsModel` on the POST path (`Converter.test_result_to_testrun_result_post_model`). + +**Entry point:** `ApiClientWorker.__load_test_result` → always POST. + +Debug log: + +```text +Finalized test result via sendTestResults for (resultId=) +``` + +### 2. PUT = fixture setup/teardown only (`importRealtime=true`) + +After the session, when tests were already sent per-test: + +1. `AdapterManager.write_tests` → `ApiClientWorker.update_test_results` +2. PUT body contains **only** `setupResults` and `teardownResults` (`Converter.convert_test_result_with_all_setup_and_teardown_steps_to_test_results_id_put_request`) +3. **Must not** include: `stepResults`, `statusCode`, `duration`, `message`, `trace`, or fields copied from GET + +Rationale: GET returns flat `StepResultApiModel` references without nested children. Re-sending them in PUT would **overwrite** the full step tree written during POST. + +Unit test: `tests/client/test_converter_update_test_results.py`. + +### 3. Never use PUT to change final status + +- Do not PUT `statusCode` / outcome to finalize a test. +- Do not implement `findInProgress` → PUT finalize (removed; was a workaround before TMS/create merge was reliable). + +### 4. Bulk at run end must not double-send + +When `importRealtime=false` and a test was already finalized at test finish (e.g. Sync Storage master path stores `externalId → resultId` in `AdapterManager.__test_result_map`): + +- `write_tests(..., finalized_external_ids=...)` **skips** `sendTestResults` for those external IDs +- Optionally refreshes autotest metadata only + +Info log: + +```text +Bulk import: skip sendTestResults for (already finalized at test finish) +``` + +See [mode0-duplicate-results-bulk-after-realtime.md](./mode0-duplicate-results-bulk-after-realtime.md). + +--- + +## Flow summary + +### `importRealtime=true` (per-test upload) + +```text +pytest_runtest_logfinish + → write_test → write_test (autotest) + __load_test_result (sendTestResults) + +pytest_sessionfinish + → update_test_results → PUT setup/teardown only (no stepResults, no status) +``` + +### `importRealtime=false` (bulk at end) + +```text +pytest_runtest_logfinish → buffer in AdapterManager.__test_results + (Sync Storage master: sendTestResults immediately + store in __test_result_map) + +pytest_sessionfinish + → write_tests → sendTestResults for buffered tests + → skip sendTestResults for keys already in __test_result_map +``` + +--- + +## Affected code + +| File | Role | +|------|------| +| `client/api_client.py` | `__load_test_result` (POST), `update_test_results` (PUT fixtures), `write_tests` (bulk + skip) | +| `client/converter.py` | `test_result_to_testrun_result_post_model`, `convert_test_result_with_all_setup_and_teardown_steps_to_test_results_id_put_request` | +| `services/adapter_manager.py` | `__test_result_map`, `write_tests` / `__write_tests_after_all` | + +--- + +## Regression checklist + +**Good:** + +- Log `Finalized test result via sendTestResults` at test finish +- With `importRealtime=true`: nested steps still on test result after session (PUT did not touch `stepResults`) +- With bulk + Sync Storage: log `Bulk import: skip sendTestResults` for tests finalized at test finish +- One finalized row per autotest per run (no TP + orphan duplicate) + +**Bad (bug is back):** + +- `Updated existing test result` / PUT with final status at test finish +- PUT body includes `stepResults` from GET in `update_test_results` +- Second `sendTestResults` for the same test in one run without skip diff --git a/testit-adapter-behave/setup.py b/testit-adapter-behave/setup.py index 4fc079b6..17a08274 100644 --- a/testit-adapter-behave/setup.py +++ b/testit-adapter-behave/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = "5.1.4" +VERSION = "5.1.5" setup( name='testit-adapter-behave', diff --git a/testit-adapter-nose/setup.py b/testit-adapter-nose/setup.py index 9e346ccd..05b3fe8a 100644 --- a/testit-adapter-nose/setup.py +++ b/testit-adapter-nose/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -VERSION = "5.1.4" +VERSION = "5.1.5" setup( name='testit-adapter-nose', diff --git a/testit-adapter-pytest/docs/displayName-flow.md b/testit-adapter-pytest/docs/displayName-flow.md new file mode 100644 index 00000000..b982db55 --- /dev/null +++ b/testit-adapter-pytest/docs/displayName-flow.md @@ -0,0 +1,285 @@ +# Схема проставления `displayName` (pytest-адаптер) + +Документ описывает **текущее** поведение адаптера: как значение из `@testit.displayName` попадает в прогон, что обновляется в карточке **автотеста** в TMS и что хранится в **тест-результате**. + +--- + +## Термины в Test IT + +| Понятие в адаптере | Декоратор / поле | Поле в API (автотест) | Где видно в TMS | +|---|---|---|---| +| Внутреннее имя автотеста | `@testit.displayName` → `test_displayname` | `name` в create/update autotest | Имя автотеста, связь с результатами прогона | +| Имя в карточке автотеста | `@testit.title` → `test_title` | `title` | Заголовок в карточке автотеста | +| Уникальный ключ автотеста | `@testit.externalId` → `test_external_id` | `external_id` | Идентификация одного автотеста в проекте | +| Параметры прогона | `item.callspec.params` | `parameters` в test run result | Параметры конкретного запуска | + +**Важно:** `displayName` — это не отдельное поле тест-результата в API. Адаптер передаёт разрешённое имя в `TestResult.autotest_name`, а дальше оно уходит в **`name` автотеста** при create/update и косвенно связывается с результатом через `auto_test_external_id`. + +--- + +## Общая схема (один запуск теста) + +```mermaid +sequenceDiagram + participant Pytest as pytest Item + participant Listener as TmsListener + participant Utils as utils.form_test + participant TR as TestResult + participant API as ApiClientWorker + participant TMS as Test IT TMS + + Pytest->>Listener: pytest_runtest_protocol(item) + Listener->>Utils: form_test(item) + Utils->>Utils: __get_display_name_from(item) + Note over Utils: подстановка {param} из callspec + Utils-->>Listener: ExecutableTest.name + Note over Listener: прогон setup / call / teardown + Listener->>Listener: pytest_runtest_logfinish + Listener->>TR: convert_executable_test_to_test_result_model + Note over TR: set_autotest_name(name) + Listener->>API: write_test(test_result) + API->>TMS: search autotest by external_id + alt автотест не найден + API->>TMS: create autotest (name = autotest_name) + else автотест найден + API->>TMS: update autotest (name = autotest_name) + end + API->>TMS: post test run result (auto_test_external_id) +``` + +--- + +## Этап 1. Декоратор `@testit.displayName` + +**Файл:** `testit_python_commons/decorators.py` + +```python +@testit.displayName("{header}") +def test_1(...): + ... +``` + +- На функцию (до обёрток `inner`) записывается атрибут **`function.test_displayname`** — строка-шаблон, например `"{header}"`. +- Декоратор возвращает обёртку `inner(function)` (sync/async wrapper), у которой через `@wraps` сохраняется цепочка `__wrapped__`; атрибут `test_displayname` остаётся на объектах в цепочке обёрток. + +Шаблон **не подставляется** на этапе импорта модуля — только при формировании теста перед прогоном. + +--- + +## Этап 2. Коллекция тестов (`pytest_collection_modifyitems`) + +**Файл:** `testit_adapter_pytest/listener.py` → `__get_separation_of_tests` + +На этом этапе для **каждого** `pytest.Item` вычисляется только **`item.test_external_id`** (для фильтрации в `adapterMode=0`): + +| Условие | Значение `item.test_external_id` | +|---|---| +| Есть `@testit.externalId` на функции | Шаблон с подстановкой параметров | +| Нет декоратора | `sha256(parent.nodeid + function.__name__)` | + +Для параметризованных тестов **без** `@testit.externalId` все варианты получают **один и тот же** `external_id` (имя функции и parent nodeid общие). + +**`displayName` на этапе коллекции не вычисляется** и на `item` не сохраняется. + +--- + +## Этап 3. Старт прогона теста (`pytest_runtest_protocol`) + +**Файл:** `testit_adapter_pytest/listener.py` + +```python +self.__executable_test = utils.form_test(item) +``` + +Вызывается **один раз на вариант** parametrized-теста, до `setup` / `call` / `teardown`. + +### 3.1. `form_test(item)` + +**Файл:** `testit_adapter_pytest/utils.py` + +| Поле `ExecutableTest` | Источник | +|---|---| +| `name` | `__get_display_name_from(item)` ← **это и есть displayName** | +| `external_id` | `__get_external_id_from(item)` | +| `title` | `@testit.title` (если есть) | +| `parameters` | `item.callspec.params` (строками) | +| … | namespace, classname, links, … | + +### 3.2. Разрешение `displayName` + +**`__get_display_name_from(item)`:** + +1. Ищет `test_displayname` через `__search_attribute(item, 'test_displayname')`: + - сначала `item.function` + - затем `item.cls` + - **`item.test_displayname` не проверяется** +2. Если декоратора нет — fallback: docstring функции или `function.__name__`. +3. Если шаблон есть — `collect_parameters_in_string_attribute(template, get_all_parameters(item))`. + +**`get_all_parameters(item)`:** + +1. `item.test_properties` (если есть; заполняется обёрткой `inner` при первом запуске — ограниченно). +2. `item.callspec.params` — основной источник для `@pytest.mark.parametrize`. +3. Для dict-параметров ключи верхнего уровня «разворачиваются» в корень (`__expand_dict_parameters`). + +**Подстановка плейсхолдеров** (`collect_parameters_in_string_attribute`): + +- Ищет `{имя}` в шаблоне. +- Берёт значение из `get_parameter` / `callspec`. +- Заменяет в строке; при отсутствии ключа пишет error в лог, плейсхолдер остаётся. + +**Пример:** `@testit.displayName("{header}")` + parametrization `header='3'` → `ExecutableTest.name == "3"`. + +### 3.3. Разрешение `externalId` (связь с displayName) + +**`__get_external_id_from(item)`** — та же схема поиска атрибута на `function` / `cls`: + +| Условие | `external_id` | +|---|---| +| Нет `@testit.externalId` | `sha256(parent.nodeid + function.__name__)` — **общий для всех параметров** | +| Есть `@testit.externalId("{header}\|{whoIs}")` | Уникальный ID после подстановки | + +**`item.test_external_id`**, посчитанный на коллекции, при `form_test` **не используется** (поиск только на function/cls). + +--- + +## Этап 4. Во время выполнения теста + +### Динамическое переименование + +**`testit.addDisplayName(...)`** → hook `add_display_name` в listener: + +```python +self.__executable_test.name = test_display_name +``` + +Перезаписывает имя **только для текущего** `ExecutableTest` до отправки в TMS. + +### Параметры + +`__get_parameters_from(item)` кладёт `callspec.params` в `TestResult.parameters` — это **параметры результата прогона**, не путать с подстановкой в `displayName`. + +--- + +## Этап 5. Завершение теста (`pytest_runtest_logfinish`) + +```python +self.__adapter_manager.write_test( + utils.convert_executable_test_to_test_result_model(self.__executable_test) +) +``` + +**`convert_executable_test_to_test_result_model`:** + +```python +TestResult() + .set_external_id(executable_test.external_id) + .set_autotest_name(executable_test.name) # ← разрешённый displayName + .set_title(executable_test.title) # ← отдельное поле title + ... +``` + +Имя из `displayName` попадает в **`autotest_name`**, не в `title`. + +--- + +## Этап 6. Отправка в TMS (`ApiClientWorker.write_test`) + +Зависит от `importRealtime` (по умолчанию в 4.x — `false`, буфер; при `true` — сразу после каждого теста). Логика **одинаковая** для одного теста. + +### 6.1. Поиск автотеста + +По `project_id` + `external_id` из `TestResult`. + +### 6.2. Create или Update автотеста + +| Операция | Модель API | Поле имени | +|---|---|---| +| Create | `CreateAutoTestRequest` / `AutoTestCreateApiModel` | `name=test_result.get_autotest_name()` | +| Update | `UpdateAutoTestRequest` / `AutoTestUpdateApiModel` | `name=test_result.get_autotest_name()` | + +Также передаются: `title`, `namespace`, `classname`, `description`, `links`, `labels`, `tags`, шаги и т.д. + +**Вывод: `displayName` при каждой отправке обновляет поле `name` автотеста в TMS** (create или update). + +### 6.3. Создание тест-результата в прогоне + +**`test_result_to_testrun_result_post_model`:** + +- `auto_test_external_id` — связь с автотестом; +- статус, длительность, шаги, attachments, `parameters`, `links` (result links); +- **отдельного поля `displayName` / `name` в модели результата нет**. + +В UI прогона имя результата обычно берётся из **текущего автотеста**, связанного по `external_id`. + +--- + +## Меняется ли `displayName` в автотесте? + +| Ситуация | Поведение | +|---|---| +| Один `external_id`, несколько parametrized-вариантов с разным `{header}` | Один автотест в TMS; при каждом `write_test` поле **`name` перезаписывается** последним прогнанным значением. В прогоне у всех результатов может отображаться **последнее** имя (например `8`). | +| У каждого варианта свой `@testit.externalId("{header}\|...")` | Несколько автотестов; у каждого своё `name` (`1`, `2`, `3`, …). | +| Повторные прогоны того же `external_id` | Update автотеста: `name` снова ставится из текущего разрешённого `displayName`. | +| Вызов `testit.addDisplayName` в теле теста | В TMS уйдёт переопределённое имя для этого запуска. | + +**`title` (`@testit.title`)** обновляется отдельно и не подменяет `displayName`; если `title` не задан, в документации TMS для карточки может использоваться имя из `displayName` (логика на стороне TMS). + +--- + +## Режимы отправки + +| `importRealtime` | Когда вызывается `write_test` | Влияние на displayName | +|---|---|---| +| `true` | После каждого теста (`pytest_runtest_logfinish`) | Имя фиксируется на момент окончания этого теста; update автотеста сразу. | +| `false` | Буфер в `AdapterManager.__test_results`, массовая отправка в `pytest_sessionfinish` | Для каждого буферизованного теста своё `autotest_name` из момента `form_test`; порядок update автотестов в bulk — по порядку в списке. | + +При включённом **Sync Storage** (не legacy workflow) возможна дополнительная ветка с отправкой статуса `InProgress` до финального результата; **`autotest_name` при этом не меняет схему** — финальная запись идёт через тот же `write_test` / bulk. + +--- + +## Сводная таблица: что куда попадает + +``` +@testit.displayName("{header}") + │ + ▼ pytest_runtest_protocol → form_test + │ +ExecutableTest.name = "3" + │ + ▼ logfinish → TestResult.autotest_name + │ + ├─► POST/PUT Autotest.name = "3" (карточка автотеста, обновляется) + │ + └─► POST TestRunResult + auto_test_external_id = + (имя в модели результата нет; в UI — через автотест) +``` + +--- + +## Рекомендация для параметризованных тестов + +Если нужны разные отображаемые имена (`1`, `2`, `3`, …) **одновременно** в одном прогоне: + +```python +@testit.displayName("{header}") +@testit.externalId("{header}|{whoIs}") # уникальный external_id обязателен +def test_1(fleet, whoIs, entities, header): + ... +``` + +Без уникального `external_id` адаптер ведёт себя корректно с точки зрения кода, но в TMS все варианты — один автотест с перезаписываемым `name`. + +--- + +## Связанные файлы в репозитории + +| Файл | Роль | +|---|---| +| `testit_python_commons/decorators.py` | `@testit.displayName`, обёртка `inner` | +| `testit_adapter_pytest/listener.py` | хуки pytest, `form_test`, `write_test` | +| `testit_adapter_pytest/utils.py` | разрешение шаблонов, `form_test` | +| `testit_python_commons/services/adapter_manager.py` | realtime / bulk, sync storage | +| `testit_python_commons/client/api_client.py` | create/update autotest, post result | +| `testit_python_commons/client/converter.py` | маппинг в API-модели | diff --git a/testit-adapter-pytest/setup.py b/testit-adapter-pytest/setup.py index 8a5eb50a..641053cf 100644 --- a/testit-adapter-pytest/setup.py +++ b/testit-adapter-pytest/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = "5.1.4" +VERSION = "5.1.5" setup( name='testit-adapter-pytest', diff --git a/testit-adapter-robotframework/setup.py b/testit-adapter-robotframework/setup.py index 184a682f..7a665081 100644 --- a/testit-adapter-robotframework/setup.py +++ b/testit-adapter-robotframework/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = "5.1.4" +VERSION = "5.1.5" setup( name='testit-adapter-robotframework', diff --git a/testit-python-commons/README.md b/testit-python-commons/README.md index 6c216b3b..69828824 100644 --- a/testit-python-commons/README.md +++ b/testit-python-commons/README.md @@ -2,7 +2,9 @@ This library includes built-in support for Sync Storage, enabling coordinated test execution across multiple workers in distributed testing environments. -See [SYNC_STORAGE_INTEGRATION.md](SYNC_STORAGE_INTEGRATION.md) for detailed documentation on how the integration works. +See [sync-storage-interaction-spec.md](sync-storage-interaction-spec.md) for the protocol. + +**Test result export (POST vs PUT):** [../docs/test-result-export-contract.md](../docs/test-result-export-contract.md) # How to enable debug logging? 1. Add in **connection_config.ini** file from the root directory of the project: diff --git a/testit-python-commons/realtime-import-spec.md b/testit-python-commons/realtime-import-spec.md index e16963c5..341d52b4 100644 --- a/testit-python-commons/realtime-import-spec.md +++ b/testit-python-commons/realtime-import-spec.md @@ -1,26 +1,52 @@ # Real-Time Import (`importRealtime=true`) Specification -This document describes how the Python adapter imports test results in real time and documents a known issue with nested test steps that appeared when the run finished. +This document describes how the Python adapter imports test results in real time. + +**Canonical export rules (POST vs PUT):** [../docs/test-result-export-contract.md](../docs/test-result-export-contract.md) ## Overview -When `importRealtime=true` (CLI: `--testit-import-realtime`, config: `importrealtime`), each test result is sent to Test IT immediately after the test completes. At the end of the session, the adapter performs an additional update to attach fixture setup/teardown steps that were not available during the per-test upload. +When `importRealtime=true` (CLI: `--testit-import-realtime`, config: `importrealtime`), each test result is sent to Test IT immediately after the test completes via **`sendTestResults`** (POST). At the end of the session, the adapter performs **one allowed PUT**: attach fixture setup/teardown steps only — **not** status, **not** test `stepResults`. -When `importRealtime=false`, all test results are sent once after the session finishes (`write_tests_after_all`). +When `importRealtime=false`, all test results are sent once after the session finishes (`write_tests_after_all`), except tests already finalized at test finish (see [../docs/mode0-duplicate-results-bulk-after-realtime.md](../docs/mode0-duplicate-results-bulk-after-realtime.md)). ## Real-Time Flow 1. **Per test** (`pytest_runtest_logfinish` → `AdapterManager.write_test` → `ApiClientWorker.write_test`): - Autotest metadata is created or updated (including nested steps in the autotest model). - - Test result is posted via `set_auto_test_results_for_test_run` with full nested `step_results` converted by `Converter.step_results_to_attachment_put_model_autotest_step_results_model`. + - Test result is finalized via **`sendTestResults`** (`POST /adapters/testRuns/{id}/test-results`) with full nested `step_results` (`Converter.test_result_to_testrun_result_post_model`). 2. **After session** (`pytest_sessionfinish` → `AdapterManager.write_tests` → `ApiClientWorker.update_test_results`): - - Only fixture setup/teardown steps are uploaded for tests that were already sent in real time. + - **PUT only** fixture `setup_results` / `teardown_results` for tests already sent in real time. + - PUT body **must not** include `step_results`, `status_code`, or other fields from GET. - Test result IDs are stored in `AdapterManager.__test_result_map` during the per-test upload. -## Bug: Nested Steps Disappear After Run Completion +## Correct PUT behaviour (session end) + +`update_test_results` uses `Converter.convert_test_result_with_all_setup_and_teardown_steps_to_test_results_id_put_request`, which builds a PUT request with **only**: + +- `setup_results` — recursive `AutoTestStepResultUpdateRequest` +- `teardown_results` — same + +Everything else is omitted so nested test steps from the earlier POST are not overwritten. + +```python +model = Converter.convert_test_result_with_all_setup_and_teardown_steps_to_test_results_id_put_request( + test_result) +self.__test_results_api.adapters_test_results_id_put( + id=test_result.get_test_result_id(), + adapters_test_results_id_put_request=model) +``` + +Unit test: `tests/client/test_converter_update_test_results.py`. + +--- + +## Historical: Nested Steps Disappeared After Run Completion + +The sections below describe **past** bugs and fixes. Current code follows [test-result-export-contract.md](../docs/test-result-export-contract.md). -### Symptoms +### Symptoms (historical) - Nested steps are visible on the **autotest** card (correct). - Nested steps are visible on the **test result** while the run is still in progress. @@ -85,83 +111,15 @@ The `step_results` field is omitted, so the nested test steps written during rea --- -### Cause B (Sync Storage + adapter): first test finalized without step tree - -When Sync Storage is running and the worker is **master**, the **first** completed test takes a special path in `AdapterManager.__write_test_realtime` → `on_master_no_already_in_progress`: - -1. Adapter sends **`TestResultCutApiModel`** to Sync Storage (`POST /in_progress_test_result`). - - Payload contains only: `projectId`, `autoTestExternalId`, `statusCode`, `statusType`, `startedOn`. - - **No `step_results`, no attachments, no nested data.** -2. Adapter sets outcome to `InProgress` and posts the test result to Test IT via `_write_test_realtime_internal`. - - This POST **does** include the full nested `step_results` tree. -3. Method returns `True` — the test is **not** written again with the real outcome through the adapter. - -All subsequent tests skip step 1–3 (Sync Storage `is_already_in_progress` flag is set) and go through normal `_write_test_realtime_internal` with the real outcome and full steps. - -At the end of the run, CI (or tooling) typically calls: - -```bash -curl http://127.0.0.1:49152/wait-completion?testRunId=... -``` - -Sync Storage then finalizes the held in-progress result and pushes the **real** status to Test IT. It only ever received the **cut** model, so the final TMS update cannot restore nested `step_results`. That explains the observed pattern: - -| Scenario | Nested steps on test result | -|----------|----------------------------| -| First test + Sync Storage + `importRealtime=true` | Visible during run, **lost after** `wait-completion` | -| Later tests in the same run | OK (normal real-time path) | -| Without Sync Storage | OK (no finalize overwrite) | -| Single test only | Always the first → always broken with Sync Storage | - -```mermaid -sequenceDiagram - participant Pytest - participant Adapter - participant SyncStorage - participant TMS - - Pytest->>Adapter: test 1 finished - Adapter->>SyncStorage: POST cut model (no steps) - Adapter->>TMS: POST InProgress + full step tree - Note over Adapter: return early, no real-outcome write - - Pytest->>Adapter: test 2..N finished - Adapter->>TMS: POST real outcome + full step tree - - Pytest->>Adapter: sessionfinish / write_tests - Adapter->>TMS: PUT setup/teardown only - - Note over CI: curl wait-completion - SyncStorage->>TMS: finalize test 1 (cut data, no nested steps) - Note over TMS: nested steps on test 1 lost -``` - -#### Recommended fixes (Cause B) +### Cause B (Sync Storage + adapter): first test and Work X finalize -**Option 1 — adapter (preferred, no Sync Storage release required):** +When Sync Storage is running and the worker is **master**, the **first** completed test uses `on_master_no_already_in_progress`: -After Sync Storage completion, re-send the full test result for the held test: +1. Adapter sends **`TestResultCutApiModel`** to Sync Storage (`POST /in_progress_test_result`) — coordination only, no steps. +2. Adapter calls `_write_test_realtime_internal` with the **final** outcome → **`sendTestResults`** with full step tree (no forced InProgress POST to TMS). +3. Stores `externalId → resultId` in `__test_result_map`. -1. In `on_master_no_already_in_progress`, **store a copy** of `TestResult` with the real outcome and full `step_results` before mutating outcome to `InProgress`. -2. In `write_tests` (or after `wait_completion`): - - Call `wait_completion` on Sync Storage if the adapter owns the lifecycle, **or** document that finalize must run after external `wait-completion`. - - `PUT` the stored result to Test IT by `test_result_id` from `__test_result_map`, including recursive `step_results` via `step_results_to_auto_test_step_result_update_request`. - -**Option 2 — Sync Storage:** - -On finalize, either do not send `step_results` in the TMS PUT (update status/duration only), or accept and persist the full `AutoTestResultsForTestRunModel` (as in the Java spec) instead of `TestResultCutApiModel`. - -**Option 3 — adapter (coordination-only):** - -Send cut data to Sync Storage for coordination but still write the real outcome to TMS immediately (do not skip `_write_test_realtime_internal` with real data). Requires validation that parallel worker coordination still works. - -#### Affected Code (Cause B) - -| File | Responsibility | -|------|----------------| -| `services/adapter_manager.py` | `on_master_no_already_in_progress`, `__write_test_realtime` | -| `services/sync_storage/sync_storage_runner.py` | `send_in_progress_test_result`, `test_result_to_test_result_cut_api_model` | -| Sync Storage binary | `/wait-completion` → TMS finalize | +Sync Storage Work X may still finalize the held cut model separately at `wait-completion`; that path is independent of the adapter POST/PUT contract. See Sync Storage docs if nested steps on the **first** test are lost after run completion. --- @@ -186,5 +144,10 @@ Unit test: `tests/client/test_converter_update_test_results.py` — asserts the | Setting | Default (4.x+) | Effect | |---------|----------------|--------| -| `importRealtime` / `importrealtime` | `false` in config, real-time path used when enabled | Per-test upload + fixture update at end | -| `adapterMode` | varies | Parallel execution / Sync Storage coordination (separate from this issue) | +| `importRealtime` / `importrealtime` | `false` in config | Per-test `sendTestResults` when enabled; session PUT = fixtures only | +| `adapterMode` | varies | Test run selection; see [../docs/test-result-export-contract.md](../docs/test-result-export-contract.md) | + +## See also + +- [../docs/test-result-export-contract.md](../docs/test-result-export-contract.md) — POST vs PUT rules +- [../docs/mode0-duplicate-results-bulk-after-realtime.md](../docs/mode0-duplicate-results-bulk-after-realtime.md) — bulk skip after test-finish finalize diff --git a/testit-python-commons/setup.py b/testit-python-commons/setup.py index 97976382..67ae0633 100644 --- a/testit-python-commons/setup.py +++ b/testit-python-commons/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = "5.1.4" +VERSION = "5.1.5" setup( name='testit-python-commons', diff --git a/testit-python-commons/src/testit_python_commons/client/api_client.py b/testit-python-commons/src/testit_python_commons/client/api_client.py index 1f9973c9..305298ca 100644 --- a/testit-python-commons/src/testit_python_commons/client/api_client.py +++ b/testit-python-commons/src/testit_python_commons/client/api_client.py @@ -30,7 +30,6 @@ from testit_python_commons.client.client_configuration import ClientConfiguration from testit_python_commons.client.converter import Converter from testit_python_commons.client.helpers.bulk_autotest_helper import BulkAutotestHelper -from testit_python_commons.client.helpers.test_result_matching import pick_best_in_progress_id from typing import List from testit_python_commons.models.link import Link @@ -63,7 +62,6 @@ def __init__(self, config: ClientConfiguration): self.__workflows_api = WorkflowsApi(api_client=api_client) self.__config = config self.__status_codes = self.__get_status_codes() - self.__claimed_in_progress_ids = set() @staticmethod @adapter_logger @@ -263,15 +261,34 @@ def write_test(self, test_result: TestResult) -> str: @adapter_logger @retry - def write_tests(self, test_results: List[TestResult], fixture_containers: dict) -> None: + def write_tests( + self, + test_results: List[TestResult], + fixture_containers: dict, + finalized_external_ids: set = None) -> None: logging.debug("call __write_tests") + finalized_external_ids = finalized_external_ids or set() bulk_autotest_helper = BulkAutotestHelper(self.__autotest_api, self.__test_run_api, self.__config) create_count = 0 update_count = 0 + skip_count = 0 tests_to_link_after_create = [] for test_result in test_results: test_result = self.__add_fixtures_to_test_result(test_result, fixture_containers) + external_id = test_result.get_external_id() + + if external_id in finalized_external_ids: + skip_count += 1 + logging.info( + 'Bulk import: skip sendTestResults for %s (already finalized at test finish)', + external_id, + ) + autotests = self.__get_autotests_by_external_id(external_id) + if autotests: + self.__update_auto_test(test_result, autotests[0]) + continue + should_create_work_item = test_result.get_automatic_creation_test_cases() test_result_model = Converter.test_result_to_testrun_result_post_model( @@ -317,9 +334,10 @@ def write_tests(self, test_results: List[TestResult], fixture_containers: dict) tests_to_link_after_create.append(test_result) logging.info( - "Bulk write summary: create=%d, update=%d, total=%d", + "Bulk write summary: create=%d, update=%d, skip=%d, total=%d", create_count, update_count, + skip_count, len(test_results), ) bulk_autotest_helper.teardown() @@ -536,87 +554,9 @@ def __link_test_to_work_item(self, autotest_global_id: str, work_item_id: str) - logging.debug(f'Autotest was linked with workItem "{work_item_id}" by global id "{autotest_global_id}') - @adapter_logger - def find_in_progress_test_result_id(self, external_id: str, parameters=None): - if not external_id: - return None - - candidates = [] - for item in self.__get_test_results(): - if item is None: - continue - if getattr(item, "autotest_external_id", None) != external_id: - continue - result_id = getattr(item, "id", None) - if result_id is None: - continue - result_id_str = str(result_id) - if result_id_str in self.__claimed_in_progress_ids: - continue - try: - detail = self.get_test_result_by_id(result_id_str) - test_point_id = getattr(detail, "test_point_id", None) - parameters_meta = getattr(detail, "parameters", None) or {} - except Exception as exc: - logging.debug("getTestResult %s failed: %s", result_id_str, exc) - test_point_id = None - parameters_meta = {} - candidates.append({ - "id": result_id_str, - "has_test_point": self.__is_valid_test_point_id(test_point_id), - "parameters": parameters_meta, - }) - - chosen = pick_best_in_progress_id(candidates, parameters) - if chosen: - self.__claimed_in_progress_ids.add(chosen) - return chosen - - @staticmethod - def __is_valid_test_point_id(test_point_id) -> bool: - if test_point_id is None: - return False - value = str(test_point_id) - return bool(value) and value != "00000000-0000-0000-0000-000000000000" - - @adapter_logger - @retry - def __update_existing_test_result(self, test_result_id: str, test_result: TestResult) -> str: - existing = self.get_test_result_by_id(test_result_id) - model = Converter.convert_test_result_model_to_test_results_id_put_request(existing) - - outcome = test_result.get_outcome() - if outcome and outcome.upper() in self.__status_codes: - model.status_code = outcome - - if test_result.get_duration() is not None: - model.duration_in_ms = round(test_result.get_duration()) - if test_result.get_message() is not None: - model.message = test_result.get_message() - if test_result.get_traces() is not None: - model.trace = test_result.get_traces() - - self.__test_results_api.adapters_test_results_id_put( - id=test_result_id, - adapters_test_results_id_put_request=model, - ) - logging.debug( - 'Updated existing test result "%s" for autotest "%s"', - test_result_id, - test_result.get_autotest_name(), - ) - return test_result_id - @adapter_logger @retry def __load_test_result(self, test_result: TestResult) -> str: - existing_id = self.find_in_progress_test_result_id( - test_result.get_external_id(), - test_result.get_parameters(), - ) - if existing_id: - return self.__update_existing_test_result(existing_id, test_result) - model = Converter.test_result_to_testrun_result_post_model( test_result, self.__config.get_configuration_id(), @@ -626,10 +566,19 @@ def __load_test_result(self, test_result: TestResult) -> str: id=self.__config.get_test_run_id(), auto_test_results_for_test_run_model=[model]) - logging.debug(f'Result of the autotest "{test_result.get_autotest_name()}" was set ' - f'in the test run "{self.__config.get_test_run_id()}"') + test_result_id = Converter.get_test_result_id_from_testrun_result_post_response(response) + logging.debug( + 'Finalized test result via sendTestResults for %s (resultId=%s)', + test_result.get_external_id(), + test_result_id, + ) + logging.debug( + 'Result of the autotest "%s" was set in the test run "%s"', + test_result.get_autotest_name(), + self.__config.get_test_run_id(), + ) - return Converter.get_test_result_id_from_testrun_result_post_response(response) + return test_result_id @adapter_logger @retry @@ -638,6 +587,8 @@ def get_test_result_by_id(self, test_result_id: str) -> TestResultResponse: @adapter_logger def update_test_results(self, fixtures_containers: dict, test_result_ids: dict) -> None: + # PUT only setup/teardown fixtures — never status or step_results. + # See docs/test-result-export-contract.md test_results = Converter.fixtures_containers_to_test_results_with_all_fixture_step_results( fixtures_containers, test_result_ids) diff --git a/testit-python-commons/src/testit_python_commons/services/adapter_manager.py b/testit-python-commons/src/testit_python_commons/services/adapter_manager.py index 28beddca..e1bd504e 100644 --- a/testit-python-commons/src/testit_python_commons/services/adapter_manager.py +++ b/testit-python-commons/src/testit_python_commons/services/adapter_manager.py @@ -189,10 +189,9 @@ def on_master_no_already_in_progress(self, test_result: TestResult) -> bool: self.__sync_storage_runner.set_is_already_in_progress(True) try: - # Keep final outcome. Prefer PUT on existing TP-bound InProgress (mode=0); - # never invent a second InProgress via setAutoTestResultsForTestRun. + # Final status via sendTestResults (create); never change status via PUT. logging.debug( - "SyncStorage accepted %s; write final status (update existing if present)", + "SyncStorage accepted %s; sendTestResults with final status", test_result.get_external_id(), ) self._write_test_realtime_internal(test_result) @@ -263,7 +262,11 @@ def __write_tests_after_all(self) -> None: for test_result in self.__test_results: test_result.set_automatic_creation_test_cases(should_create_work_item) - self.__api_client.write_tests(self.__test_results, fixtures) + self.__api_client.write_tests( + self.__test_results, + fixtures, + finalized_external_ids=set(self.__test_result_map.keys()), + ) @adapter_logger def load_attachments(self, attach_paths): diff --git a/testit-python-commons/tests/client/test_find_in_progress.py b/testit-python-commons/tests/client/test_find_in_progress.py index 50360ce4..661ddb0a 100644 --- a/testit-python-commons/tests/client/test_find_in_progress.py +++ b/testit-python-commons/tests/client/test_find_in_progress.py @@ -1,6 +1,3 @@ -import pytest - -from testit_python_commons.client.api_client import ApiClientWorker from testit_python_commons.client.helpers.test_result_matching import ( normalize_parameters, parameters_empty, @@ -52,135 +49,3 @@ def test_non_parametrized_picks_empty_with_tp(self): {"id": "bound", "has_test_point": True, "parameters": {}}, ] assert pick_best_in_progress_id(candidates, None) == "bound" - - -class TestFindInProgressAndUpdate: - @pytest.fixture - def worker(self, mocker): - mocker.patch.object( - ApiClientWorker, - "_ApiClientWorker__get_status_codes", - return_value=["PASSED", "FAILED", "INPROGRESS", "SKIPPED", "BLOCKED"], - ) - mocker.patch.object(ApiClientWorker, "_ApiClientWorker__get_api_client_configuration") - mocker.patch.object(ApiClientWorker, "_ApiClientWorker__get_api_client") - mocker.patch("testit_python_commons.client.api_client.TestRunsApi") - mocker.patch("testit_python_commons.client.api_client.AutoTestsApi") - mocker.patch("testit_python_commons.client.api_client.AttachmentsApi") - mocker.patch("testit_python_commons.client.api_client.TestResultsApi") - mocker.patch("testit_python_commons.client.api_client.WorkItemsApi") - mocker.patch("testit_python_commons.client.api_client.ProjectsApi") - mocker.patch("testit_python_commons.client.api_client.WorkflowsApi") - - config = mocker.Mock() - config.get_url.return_value = "https://tms.example" - config.get_private_token.return_value = "token" - config.get_cert_validation.return_value = True - config.get_project_id.return_value = "proj" - config.get_test_run_id.return_value = "run-1" - config.get_configuration_id.return_value = "cfg-1" - config.get_proxy.return_value = None - - return ApiClientWorker(config) - - def test_find_in_progress_prefers_valid_test_point(self, worker, mocker): - orphan = mocker.Mock(id="orphan-id", autotest_external_id="ext-1") - bound = mocker.Mock(id="bound-id", autotest_external_id="ext-1") - mocker.patch.object( - worker, - "_ApiClientWorker__get_test_results", - return_value=[orphan, bound], - ) - mocker.patch.object( - worker, - "get_test_result_by_id", - side_effect=lambda rid: mocker.Mock(**{ - "orphan-id": {"test_point_id": None, "parameters": {}}, - "bound-id": {"test_point_id": "tp-1", "parameters": {}}, - }[rid]), - ) - - assert worker.find_in_progress_test_result_id("ext-1") == "bound-id" - - def test_find_exact_parameters_among_multiple(self, worker, mocker): - a = mocker.Mock(id="res-a", autotest_external_id="ext-1") - b = mocker.Mock(id="res-b", autotest_external_id="ext-1") - mocker.patch.object( - worker, - "_ApiClientWorker__get_test_results", - return_value=[a, b], - ) - mocker.patch.object( - worker, - "get_test_result_by_id", - side_effect=lambda rid: mocker.Mock(**{ - "res-a": {"test_point_id": "tp-a", "parameters": {"x": "1"}}, - "res-b": {"test_point_id": "tp-b", "parameters": {"x": "2"}}, - }[rid]), - ) - - assert worker.find_in_progress_test_result_id("ext-1", {"x": "2"}) == "res-b" - - def test_find_empty_wi_params_fallback_and_claim(self, worker, mocker): - a = mocker.Mock(id="res-a", autotest_external_id="ext-1") - b = mocker.Mock(id="res-b", autotest_external_id="ext-1") - mocker.patch.object( - worker, - "_ApiClientWorker__get_test_results", - return_value=[a, b], - ) - mocker.patch.object( - worker, - "get_test_result_by_id", - side_effect=lambda rid: mocker.Mock(**{ - "res-a": {"test_point_id": "tp-a", "parameters": {}}, - "res-b": {"test_point_id": "tp-b", "parameters": {}}, - }[rid]), - ) - - first = worker.find_in_progress_test_result_id("ext-1", {"browser": "chrome"}) - second = worker.find_in_progress_test_result_id("ext-1", {"browser": "firefox"}) - - assert first == "res-a" - assert second == "res-b" - assert first != second - - def test_find_returns_none_for_incompatible_only(self, worker, mocker): - a = mocker.Mock(id="res-a", autotest_external_id="ext-1") - mocker.patch.object( - worker, - "_ApiClientWorker__get_test_results", - return_value=[a], - ) - mocker.patch.object( - worker, - "get_test_result_by_id", - return_value=mocker.Mock(test_point_id="tp-a", parameters={"x": "1"}), - ) - - assert worker.find_in_progress_test_result_id("ext-1", {"x": "2"}) is None - - def test_load_test_result_updates_existing(self, worker, mocker): - mocker.patch.object( - worker, - "find_in_progress_test_result_id", - return_value="existing-id", - ) - update = mocker.patch.object( - worker, - "_ApiClientWorker__update_existing_test_result", - return_value="existing-id", - ) - create_post = mocker.patch.object( - worker._ApiClientWorker__test_run_api, - "adapters_test_runs_id_test_results_post", - ) - - test_result = mocker.Mock() - test_result.get_external_id.return_value = "ext-1" - test_result.get_parameters.return_value = {"x": "1"} - - assert worker._ApiClientWorker__load_test_result(test_result) == "existing-id" - worker.find_in_progress_test_result_id.assert_called_once_with("ext-1", {"x": "1"}) - update.assert_called_once() - create_post.assert_not_called() diff --git a/testit-python-commons/tests/client/test_update_existing_test_result.py b/testit-python-commons/tests/client/test_update_existing_test_result.py new file mode 100644 index 00000000..42907d1c --- /dev/null +++ b/testit-python-commons/tests/client/test_update_existing_test_result.py @@ -0,0 +1,69 @@ +from unittest.mock import Mock + +import pytest + +from testit_python_commons.client.api_client import ApiClientWorker +from testit_python_commons.models.outcome_type import OutcomeType +from testit_python_commons.models.step_result import StepResult +from testit_python_commons.models.test_result import TestResult + + +@pytest.fixture +def worker(mocker): + mocker.patch.object( + ApiClientWorker, + "_ApiClientWorker__get_status_codes", + return_value=["PASSED", "FAILED", "INPROGRESS", "SKIPPED", "BLOCKED"], + ) + mocker.patch.object(ApiClientWorker, "_ApiClientWorker__get_api_client_configuration") + mocker.patch.object(ApiClientWorker, "_ApiClientWorker__get_api_client") + mocker.patch("testit_python_commons.client.api_client.TestRunsApi") + mocker.patch("testit_python_commons.client.api_client.AutoTestsApi") + mocker.patch("testit_python_commons.client.api_client.AttachmentsApi") + mocker.patch("testit_python_commons.client.api_client.TestResultsApi") + mocker.patch("testit_python_commons.client.api_client.WorkItemsApi") + mocker.patch("testit_python_commons.client.api_client.ProjectsApi") + mocker.patch("testit_python_commons.client.api_client.WorkflowsApi") + + config = mocker.Mock() + config.get_url.return_value = "https://tms.example" + config.get_private_token.return_value = "token" + config.get_cert_validation.return_value = True + config.get_project_id.return_value = "proj" + config.get_test_run_id.return_value = "run-1" + config.get_configuration_id.return_value = "cfg-1" + config.get_proxy.return_value = None + + return ApiClientWorker(config) + + +def test_load_test_result_always_uses_send_test_results(worker, mocker): + post_mock = mocker.patch.object( + worker._ApiClientWorker__test_run_api, + "adapters_test_runs_id_test_results_post", + return_value=["result-id"], + ) + put_mock = mocker.patch.object( + worker._ApiClientWorker__test_results_api, + "adapters_test_results_id_put", + ) + + step = StepResult().set_title("write step").set_outcome(OutcomeType.PASSED) + test_result = TestResult() + test_result.set_step_results([step]) + test_result.set_outcome("Passed") + test_result.set_duration(100) + test_result.set_autotest_name("test") + test_result.set_external_id("ext-1") + test_result.set_parameters({"x": "1"}) + test_result.set_status_type("Succeeded") + + assert worker._ApiClientWorker__load_test_result(test_result) == "result-id" + + post_mock.assert_called_once() + model = post_mock.call_args.kwargs["auto_test_results_for_test_run_model"][0] + assert len(model.step_results) == 1 + assert model.step_results[0].title == "write step" + assert model.status_code == "Passed" + assert model.duration == 100 + put_mock.assert_not_called() diff --git a/testit-python-commons/tests/client/test_write_tests_bulk_skip.py b/testit-python-commons/tests/client/test_write_tests_bulk_skip.py new file mode 100644 index 00000000..012b81c8 --- /dev/null +++ b/testit-python-commons/tests/client/test_write_tests_bulk_skip.py @@ -0,0 +1,83 @@ +import pytest + +from testit_python_commons.client.api_client import ApiClientWorker + + +@pytest.fixture +def worker(mocker): + mocker.patch.object( + ApiClientWorker, + "_ApiClientWorker__get_status_codes", + return_value=["PASSED", "FAILED", "INPROGRESS", "SKIPPED", "BLOCKED"], + ) + mocker.patch.object(ApiClientWorker, "_ApiClientWorker__get_api_client_configuration") + mocker.patch.object(ApiClientWorker, "_ApiClientWorker__get_api_client") + mocker.patch("testit_python_commons.client.api_client.TestRunsApi") + mocker.patch("testit_python_commons.client.api_client.AutoTestsApi") + mocker.patch("testit_python_commons.client.api_client.AttachmentsApi") + mocker.patch("testit_python_commons.client.api_client.TestResultsApi") + mocker.patch("testit_python_commons.client.api_client.WorkItemsApi") + mocker.patch("testit_python_commons.client.api_client.ProjectsApi") + mocker.patch("testit_python_commons.client.api_client.WorkflowsApi") + + config = mocker.Mock() + config.get_url.return_value = "https://tms.example" + config.get_private_token.return_value = "token" + config.get_cert_validation.return_value = True + config.get_project_id.return_value = "proj" + config.get_test_run_id.return_value = "run-1" + config.get_configuration_id.return_value = "cfg-1" + config.get_proxy.return_value = None + + return ApiClientWorker(config) + + +def test_write_tests_skips_send_for_already_finalized(worker, mocker): + bulk_helper_cls = mocker.patch("testit_python_commons.client.api_client.BulkAutotestHelper") + bulk_helper = bulk_helper_cls.return_value + mocker.patch.object(worker, "_ApiClientWorker__add_fixtures_to_test_result", side_effect=lambda tr, _: tr) + mocker.patch.object( + worker, + "_ApiClientWorker__get_autotests_by_external_id", + return_value=[mocker.Mock(id="autotest-1")], + ) + update_auto_test = mocker.patch.object(worker, "_ApiClientWorker__update_auto_test") + + finalized = mocker.Mock() + finalized.get_external_id.return_value = "ext-finalized" + finalized.get_automatic_creation_test_cases.return_value = False + + pending = mocker.Mock() + pending.get_external_id.return_value = "ext-pending" + pending.get_automatic_creation_test_cases.return_value = False + pending.get_status_type.return_value = "Succeeded" + pending.get_step_results.return_value = [] + pending.get_setup_results.return_value = [] + pending.get_teardown_results.return_value = [] + pending.get_outcome.return_value = "Passed" + pending.get_duration.return_value = 0 + pending.get_message.return_value = None + pending.get_traces.return_value = None + pending.get_result_links.return_value = [] + pending.get_parameters.return_value = {} + pending.get_properties.return_value = {} + pending.get_attachments.return_value = [] + pending.get_started_on.return_value = None + pending.get_completed_on.return_value = None + pending.get_work_item_ids.return_value = [] + + mocker.patch( + "testit_python_commons.client.api_client.Converter.prepare_to_mass_update_autotest", + return_value=mocker.Mock(), + ) + + worker.write_tests( + [finalized, pending], + {}, + finalized_external_ids={"ext-finalized"}, + ) + + update_auto_test.assert_called_once_with(finalized, mocker.ANY) + bulk_helper.add_for_create.assert_not_called() + bulk_helper.add_for_update.assert_called_once() + bulk_helper.teardown.assert_called_once() diff --git a/testit-python-commons/tests/services/test_adapter_manager.py b/testit-python-commons/tests/services/test_adapter_manager.py index 3b013832..c0c01154 100644 --- a/testit-python-commons/tests/services/test_adapter_manager.py +++ b/testit-python-commons/tests/services/test_adapter_manager.py @@ -158,6 +158,7 @@ def test_write_tests_sets_automatic_creation_for_all_buffered_results( mock_api_client_worker.write_tests.assert_called_once_with( [test_result_1, test_result_2], fixtures, + finalized_external_ids=set(), ) def test_write_test_realtime_sets_automatic_creation_for_each_test( diff --git a/update_versions.sh b/update_versions.sh index 58bcb102..b6a16288 100644 --- a/update_versions.sh +++ b/update_versions.sh @@ -1,6 +1,6 @@ #!/bin/bash -NEW_VERSION="5.1.4" +NEW_VERSION="5.1.5" echo "Updating all adapters to version: $NEW_VERSION" From b8d08a9d7cc9824ddb9c3211f92d6f91f086f774 Mon Sep 17 00:00:00 2001 From: Dmitry Ermakovich Date: Fri, 28 Aug 2026 11:17:14 +0300 Subject: [PATCH 2/3] chore: fix --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 38aad347..b173fcf8 100644 --- a/.gitignore +++ b/.gitignore @@ -60,4 +60,4 @@ pytestdebug.log .vs/ *-examples/ -.user/ \ No newline at end of file +.user/ From 92b704a79a687866c5ed209121b3b285469f0165 Mon Sep 17 00:00:00 2001 From: Dmitry Ermakovich Date: Fri, 28 Aug 2026 11:33:04 +0300 Subject: [PATCH 3/3] fix: skipped tests issue --- .../src/testit_adapter_pytest/listener.py | 4 ++++ .../src/testit_adapter_pytest/utils.py | 10 ++++++++++ testit-adapter-pytest/tests/test_utils.py | 18 ++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/testit-adapter-pytest/src/testit_adapter_pytest/listener.py b/testit-adapter-pytest/src/testit_adapter_pytest/listener.py index b945d11d..8f72f1a2 100644 --- a/testit-adapter-pytest/src/testit_adapter_pytest/listener.py +++ b/testit-adapter-pytest/src/testit_adapter_pytest/listener.py @@ -261,6 +261,10 @@ def pytest_runtest_logreport(self, report): if report.failed or report.outcome == 'rerun' or is_xfailed: self.__executable_test.traces = getattr(report, 'longreprtext', None) + skip_message = utils.get_skip_message_from_report(report) + if skip_message: + self.__executable_test.message = skip_message + self.__executable_test.duration += report.duration * 1000 @pytest.hookimpl diff --git a/testit-adapter-pytest/src/testit_adapter_pytest/utils.py b/testit-adapter-pytest/src/testit_adapter_pytest/utils.py index a3716fb4..d73fe6c7 100644 --- a/testit-adapter-pytest/src/testit_adapter_pytest/utils.py +++ b/testit-adapter-pytest/src/testit_adapter_pytest/utils.py @@ -436,5 +436,15 @@ def get_traceback(exc_traceback): return ''.join(traceback.format_tb(exc_traceback)) if exc_traceback else None +def get_skip_message_from_report(report) -> str or None: + if report is None: + return None + if not (getattr(report, 'skipped', False) or report.outcome == 'skipped'): + return None + if report.longrepr is not None: + return str(report.longrepr) + return None + + def get_message(etype, value): return '\n'.join(format_exception_only(etype, value)) if etype or value else None diff --git a/testit-adapter-pytest/tests/test_utils.py b/testit-adapter-pytest/tests/test_utils.py index e23fbc89..de79989f 100644 --- a/testit-adapter-pytest/tests/test_utils.py +++ b/testit-adapter-pytest/tests/test_utils.py @@ -4,6 +4,7 @@ collect_parameters_in_string_attribute, get_all_parameters, get_parameter, + get_skip_message_from_report, ) @@ -53,3 +54,20 @@ def test_top_level_param_not_overwritten_by_dict_expand(): params = get_all_parameters(item) assert params['user_type'] == 'from_parametrize' + + +def test_get_skip_message_from_parametrize_skip_report(): + report = type('Report', (), { + 'skipped': True, + 'outcome': 'skipped', + 'longrepr': ( + '/path/tests/test_annotations.py', + 141, + 'Skipped: bug test', + ), + })() + + assert get_skip_message_from_report(report) == ( + "('/path/tests/test_annotations.py', 141, 'Skipped: bug test')" + ) +