Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions docs/development/testing-and-quality.md
Original file line number Diff line number Diff line change
Expand Up @@ -431,6 +431,18 @@ it differs from the live server before dispatching expensive work. Stop only
that isolated runtime after its requests settle, retaining the same temporary
environment for shutdown; never restart a live server as test cleanup.

For namespace disposal, call `restart_effect_runtime(wait_for_exit=True)` and
require `stopped` before removing files. A closed listener or a zombie process
group leader does not prove that all runtime threads have exited. A PID that is
still present returns `shutdown_pending`; retain the namespace and retry after
the process exits and its parent reaps it. The ordinary restart action still
uses serving-locator retirement.

删除隔离目录前,调用 `restart_effect_runtime(wait_for_exit=True)` 并确认返回
`stopped`。端口关闭或进程组长成为僵尸进程,都不能证明其他线程已经退出。
PID 仍存在时返回 `shutdown_pending`;保留目录,等进程退出并由父进程回收后再试。
普通重启操作仍按服务 locator 的退役状态判断。

Automatic test cleanup must verify namespace ownership as well as PID/token
identity. An ordinary copy of a valid locator inside a test directory is not
permission to stop its original server. The private authenticated `runtime.ping`
Expand Down
17 changes: 1 addition & 16 deletions loopx/control_plane/effect_runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -771,22 +771,7 @@ def _stop_effect_runtime(
if not _pid_is_alive(pid):
stopped = True
break
if os.name == "posix":
# A non-child zombie cannot write, but kill(pid, 0) still sees
# it. Managed runtimes start in their own session/process group.
from ..extensions.process_runtime import _posix_owned_group_has_exited
try:
stopped = _posix_owned_group_has_exited(
pid, max(.001, deadline - time.monotonic()),
)
except (OSError, subprocess.TimeoutExpired, UnicodeError, RuntimeError):
stopped = False
if stopped:
# The child may have exited during the group snapshot,
# after the first nonblocking reap. Retire its wait status
# before returning successful namespace cleanup.
_reap_exited_runtime_child(info)
break
# A zombie leader can still have threads writing into the namespace.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] 删除该调用后,现有真实回收测试的 child 永远等 stdin;唯一关闭 stdin 的 monkeypatch 回调再也不执行。当前四模块50过1败,替换不可变 base owner 的同一测试过。请迁移退出触发、保留真实 waitpid/PID 断言;不要恢复已证伪的 ps 快捷成功或删除有效回收覆盖。

time.sleep(.05)
continue
published, published_token = _serving_token(info_path)
Expand Down
49 changes: 47 additions & 2 deletions tests/control_plane/test_effect_runtime_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,16 +33,17 @@


@pytest.mark.parametrize("replacement", [False, True])
@pytest.mark.parametrize("owns_group", [True, False] if os.name == "posix" else [False])
def test_fixture_shutdown_waits_for_process_exit_after_locator_turnover(
tmp_path, monkeypatch, replacement,
tmp_path, monkeypatch, replacement, owns_group,
):
locator = tmp_path / "runtime.json"
if replacement:
locator.write_text(json.dumps({"token": "replacement"}))
monkeypatch.setattr(effect_runtime, "_request_with_info", lambda *a, **kw: {})
with subprocess.Popen(
[sys.executable, "-c", "import time; print('ready',flush=True); time.sleep(60)"],
stdout=subprocess.PIPE, text=True, start_new_session=os.name == "posix",
stdout=subprocess.PIPE, text=True, start_new_session=owns_group,
) as process:
try:
assert process.stdout.readline().strip() == "ready"
Expand All @@ -69,6 +70,50 @@ def test_fixture_shutdown_waits_for_process_exit_after_locator_turnover(
process.wait()


@pytest.mark.skipif(sys.platform != "linux", reason="Linux thread-group leader exit")
def test_fixture_shutdown_waits_for_writer_after_thread_group_leader_exits(tmp_path, monkeypatch):
witness = tmp_path / "written-after-stop"
monkeypatch.setattr(effect_runtime, "_request_with_info", lambda *a, **kw: {})
source = """
import ctypes, os, pathlib, sys, threading
def write_after_release():
print('ready', flush=True)
sys.stdin.readline()
pathlib.Path(sys.argv[1]).write_text('worker can still write')
os._exit(0)
threading.Thread(target=write_after_release).start()
ctypes.CDLL(None).pthread_exit(None)
"""
with subprocess.Popen(
[sys.executable, "-c", source, str(witness)],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, start_new_session=True,
) as process:
try:
assert process.stdout.readline().strip() == "ready"
state = Path(f"/proc/{process.pid}/stat")
deadline = time.monotonic() + 5
while state.read_text().rsplit(")", 1)[1].split()[0] != "Z":
assert time.monotonic() < deadline, "thread-group leader did not exit"
time.sleep(.01)
info = {"pid": process.pid, "token": "original"}
stopped = effect_runtime._stop_effect_runtime(
tmp_path / "retired-locator", info, timeout=.1, wait_for_process_exit=True,
)
assert stopped["status"] == "shutdown_pending"
assert effect_runtime._pid_is_alive(process.pid)
process.stdin.write("release\n")
process.stdin.flush()
process.wait(timeout=5)
assert witness.read_text() == "worker can still write"
assert effect_runtime._stop_effect_runtime(
tmp_path / "retired-locator", info, timeout=.1, wait_for_process_exit=True,
)["status"] == "stopped"
finally:
if process.poll() is None:
process.kill()
process.wait(timeout=5)


def _effect_id(label: str) -> str:
return f"fixture-goal:fixture-agent:{_TODO_ID}:{label}"

Expand Down
Loading