ZaStoGram_desktop/Telegram/SourceFiles/tests/proxy_control_plane_truth.py
loop-uh 744109cc78 Pace MTProxy dials instead of steering them
Every session dialled a proxy on its own with nothing in between, so a cold
start or a proxy switch opened one socket per session per account in the same
millisecond. A public mtproxy answers about two parallel handshakes and
silently drops the rest, which the client read back as
client_hello_sent_no_server_hello or connected_no_mtproto_data across the
whole batch - including on a dd-secret proxy that sends no ClientHello at all
and therefore cannot be fingerprint-blocked.

Replace the machinery that was supposed to prevent this with a rate limiter
that actually does. ProxyDialLease keeps at most two unproven handshakes in
flight per proxy server, spaces the rest apart and stretches the spacing
after a run of attempts that proved nothing, so a blackholed proxy is no
longer redialled by every session on its own eight second timer.

Everything that reacted to failure by changing its own behaviour is gone:
EndpointAdmissionArbiter, EndpointLivePool, ConnectionBroker,
session_proxy_adapter, SessionProxyPort, HandshakeGate, open_scheduler,
endpoint_health, adaptive_policy and ProxyRotationManager. A fingerprint
filter is deterministic, so rotating emulated ClientHello profiles only
hands the other side more of them, and escalated recipes (fragmentation,
pacing) make the flow less browser-like rather than more - while the signal
that drove the escalation could not tell a DPI box from a proxy refusing
extra connections. The ClientHello templates themselves are untouched and
the profile is now whatever the user configured, fixed.

Also: ServerHello budget 2.5s -> 5s, mtproxy status reduces through the same
ProxyConnectionStatus path as every other proxy type, and route memory is
wired back up so an address that answered is dialled first and one that
failed is dialled last.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-07-25 16:58:05 +03:00

3259 lines
101 KiB
Python

from copy import deepcopy
from dataclasses import dataclass, field, replace
from pathlib import Path
SOURCE_DIR = Path(__file__).resolve().parents[1]
PROXY_DIR = SOURCE_DIR / "mtproto" / "proxy"
CONTROL_CPP = PROXY_DIR / "control_plane.cpp"
CHECK_CPP = PROXY_DIR / "check.cpp"
CAPABILITIES_CPP = PROXY_DIR / "capabilities.cpp"
ENDPOINT_HEALTH_LIFECYCLE_CPP = (
PROXY_DIR / "mtproxy" / "endpoint_health_lifecycle.cpp")
ENDPOINT_HEALTH_CAPABILITIES_CPP = (
PROXY_DIR / "mtproxy" / "endpoint_health_capabilities.cpp")
PROXY_ENDPOINT_CONTEXT_CPP = PROXY_DIR / "proxy_endpoint_context.cpp"
SESSION_CONNECTION_CPP = (
SOURCE_DIR / "mtproto" / "session" / "private" / "connection.cpp")
INSTANCE_CPP = SOURCE_DIR / "mtproto" / "instance" / "mtp_instance.cpp"
TLS_SOCKET_CPP = PROXY_DIR / "mtproxy" / "tls_socket.cpp"
TLS_SOCKET_RECORDS_CPP = PROXY_DIR / "mtproxy" / "tls_socket_records.cpp"
NONE = "none"
FAILED = "failed"
CONNECTED = "connected"
CHECKING = "checking"
NO_SERVERHELLO = "client_hello_sent_no_server_hello"
NO_APPDATA = "server_hello_ok_no_appdata"
NO_MTPROTO = "server_hello_ok_no_mtproto_data"
MTP_TIMEOUT_AFTER_DATA = "mtp_receive_timeout_after_data"
HMAC_MISMATCH = "server_hello_hmac_mismatch"
RELAY = "relay"
HANDSHAKE = "handshake"
FAKETLS_APPDATA = "faketls_appdata"
INSERTED = "Inserted"
ALREADY_PROVEN = "AlreadyProven"
MISSING_ADMISSION = "MissingAdmission"
STALE_GENERATION = "StaleGeneration"
STALE_ATTEMPT = "StaleAttempt"
MISSING_OR_DUPLICATE = "MissingOrDuplicate"
RETIRED_WITH_SURVIVORS = "RetiredWithSurvivors"
RETIRED_FINAL = "RetiredFinal"
TERMINAL_ATTEMPT = "TerminalAttempt"
ATTEMPT_HARD_TTL = 120000
OPENING_PRESSURE_COOLDOWN = 15000
ABC_ENDPOINT = "151.247.209.166.sslip.io:45632"
TCP_TIMEOUT = "tcp_connect_timeout"
CONNECTED_NO_MTPROTO = "connected_no_mtproto_data"
DNS_FAILED = "dns_failed"
CANCELLED = "cancelled"
REMOTE_CLOSED = "remote_closed"
EMPTY = "empty"
RESERVED = "reserved"
OPENING = "opening"
LIVE = "live"
CLOSING = "closing"
SLOT_WAIT = "slot"
CAPACITY_WAIT = "capacity"
CLOSING_WAIT = "closing"
RESERVE_ACTION = "reserve"
RECLAIM_ACTION = "reclaim"
WAIT_ACTION = "wait"
ORDINARY = "ordinary"
RECLAIMED_MAIN_RESUME = "reclaimed_main_resume"
NO_TRANSFER_ADMISSION = "none"
MAIN_RELAY_PROOF = "main_relay_proof"
RELEASED_CONTINUATION = "released_continuation"
CONTINUATION_REQUESTED = "requested"
AWAITING_RELEASE = "awaiting_release"
CONTINUATION_RELEASED = "released"
RECLAIM_REQUESTED = "requested"
RECLAIM_AUTHORIZED = "authorized"
FOREGROUND_RECOVERY = "foreground_recovery"
DEMAND_BOOTSTRAP = "demand_bootstrap"
BACKGROUND_DUTY = "background_duty"
MAIN = "main"
MEDIA = "media"
UPLOAD = "upload"
PROXY_CHECK = "proxy_check"
LIVE_SLOT_COUNT = 4
CAPACITY_PRESSURE_REASONS = {
TCP_TIMEOUT,
NO_SERVERHELLO,
NO_APPDATA,
NO_MTPROTO,
CONNECTED_NO_MTPROTO,
}
@dataclass(frozen=True)
class Attempt:
proxy_generation: int = 0
proxy_epoch: int = 0
success_epoch: int = 0
attempt_id: int = 0
probe: bool = False
@dataclass(frozen=True)
class Status:
phase: str = NONE
reason: str = NONE
error: bool = False
attempt: Attempt = Attempt()
terminal_until: int = 0
success_until: int = 0
@dataclass(frozen=True)
class Fact:
status: Status = Status()
success_scope: str = NONE
@dataclass(frozen=True, order=True)
class RelayProofIdentity:
runtime_id: int = 0
proxy_generation: int = 0
attempt_id: int = 0
@dataclass(frozen=True)
class EndpointAttemptState:
runtime_id: int = 0
proxy_generation: int = 0
started_at: int = 0
@dataclass(frozen=True)
class RelayProofState:
proven_at: int = 0
@dataclass(frozen=True)
class RelayReport:
endpoint: str
runtime_id: int = 0
proxy_generation: int = 0
attempt_id: int = 0
proxy_epoch: int = 0
success_epoch: int = 0
started_at: int = 0
@dataclass
class EndpointState:
generations: dict = field(default_factory=dict)
attempt_starts: dict = field(default_factory=dict)
opening_pressure_reason: str = NONE
opening_pressure_retry_until: int = 0
proxy_epoch: int = 1
success_epoch: int = 0
relay_proofs: dict = field(default_factory=dict)
last_success_at: int = 0
last_relay_success_at: int = 0
relay_proven: bool = False
healthy: bool = False
route_state: str = "unknown"
route_success_count: int = 0
route_failure_count: int = 0
cooldown_until: int = 0
cooldown_count: int = 0
rotation_count: int = 0
capability_success_count: int = 0
capability_failure_count: int = 0
relay_capability_failure_count: int = 0
capability_relay_proven: bool = False
last_failure: str = NONE
@dataclass
class CanonicalEndpointStore:
states: dict = field(default_factory=dict)
def state(self, endpoint):
return self.states.setdefault(endpoint, EndpointState())
@dataclass
class BrokerRequest:
active: bool = True
has_context: bool = True
admission: bool = False
admission_lease_active: bool = False
admission_in_progress: bool = False
start_scheduled: bool = False
@dataclass(frozen=True, order=True)
class LiveSlotKey:
endpoint: str
index: int
incarnation: int
@dataclass(frozen=True)
class LiveTicket:
runtime_id: int
ticket_id: int
revision: int
proxy_generation: int = 1
purpose: str = ORDINARY
@dataclass
class LiveSlot:
phase: str = EMPTY
incarnation: int = 0
ticket: LiveTicket | None = None
attempt: RelayProofIdentity | None = None
use: str = MAIN
live_since: int = 0
@dataclass(frozen=True)
class CapacityProbe:
key: LiveSlotKey
attempt: RelayProofIdentity
target: int
baseline: tuple[LiveSlotKey, ...]
@dataclass
class LiveReclaim:
key: LiveSlotKey
incumbent: RelayProofIdentity
successor: LiveTicket | None
stage: str = RECLAIM_REQUESTED
continuation_required: bool = False
@dataclass
class DemandTransferContinuation:
source_key: LiveSlotKey
source_attempt: RelayProofIdentity
successor: LiveTicket
runtime_id: int
proxy_generation: int
use: str
stage: str = CONTINUATION_REQUESTED
@dataclass(frozen=True)
class LiveReleaseResult:
slot_released: bool = False
successor: LiveTicket | None = None
continuation_changed: bool = False
@dataclass
class LivePool:
slots: list[LiveSlot] = field(default_factory=lambda: [
LiveSlot() for _ in range(LIVE_SLOT_COUNT)
])
opening_ticket: LiveTicket | None = None
opening_attempt: RelayProofIdentity | None = None
probe: CapacityProbe | None = None
reclaim: LiveReclaim | None = None
demand_transfer_continuation: DemandTransferContinuation | None = None
proven_lower_bound: int = 0
learned_limit: int | None = None
last_incarnation: int = 0
def slot_key(endpoint, index, slot):
return LiveSlotKey(endpoint, index, slot.incarnation)
def find_live_slot(pool, key):
if (
not key.endpoint
or key.index < 0
or key.index >= LIVE_SLOT_COUNT
):
return None
slot = pool.slots[key.index]
return slot if slot.incarnation == key.incarnation else None
def nonempty_slot_count(pool):
return sum(slot.phase != EMPTY for slot in pool.slots)
def complete_live_baseline(pool, endpoint):
return tuple(sorted(
slot_key(endpoint, index, slot)
for index, slot in enumerate(pool.slots)
if slot.phase == LIVE and slot.attempt is not None
))
def reset_learning_if_empty(pool):
if (
any(slot.phase != EMPTY for slot in pool.slots)
or pool.demand_transfer_continuation is not None
):
return
pool.proven_lower_bound = 0
pool.learned_limit = None
pool.probe = None
pool.reclaim = None
def continuation_matches(
pool,
endpoint,
ticket,
use,
stage=CONTINUATION_RELEASED):
continuation = pool.demand_transfer_continuation
return (
continuation is not None
and continuation.stage == stage
and continuation.source_key.endpoint == endpoint
and continuation.source_key.incarnation > 0
and continuation.successor == ticket
and continuation.runtime_id == ticket.runtime_id
and continuation.proxy_generation == ticket.proxy_generation
and continuation.use == use
and use in {MEDIA, UPLOAD}
and continuation.source_attempt.runtime_id == ticket.runtime_id
and continuation.source_attempt.proxy_generation
== ticket.proxy_generation
and continuation.source_attempt.attempt_id != 0
)
def transfer_admission_matches(pool, endpoint, ticket, use, basis):
if use not in {MEDIA, UPLOAD}:
return basis == NO_TRANSFER_ADMISSION
if basis == MAIN_RELAY_PROOF:
return True
if basis == RELEASED_CONTINUATION:
return continuation_matches(pool, endpoint, ticket, use)
return False
def pool_wait_reason(pool, endpoint, ticket):
if pool.reclaim:
return (
CLOSING_WAIT
if pool.reclaim.successor == ticket
else SLOT_WAIT
)
continuation = pool.demand_transfer_continuation
if continuation and continuation.successor != ticket:
return SLOT_WAIT
if continuation and continuation.stage != CONTINUATION_RELEASED:
return CLOSING_WAIT
if pool.opening_ticket or pool.opening_attempt:
return SLOT_WAIT
empty = next((
index
for index, slot in enumerate(pool.slots)
if slot.phase == EMPTY
), None)
if empty is None:
return SLOT_WAIT
occupied = nonempty_slot_count(pool)
proven = max(0, min(pool.proven_lower_bound, LIVE_SLOT_COUNT))
if occupied < proven or (not proven and not occupied):
return NONE
if pool.learned_limit is not None and occupied >= pool.learned_limit:
return CAPACITY_WAIT
if pool.probe or proven >= LIVE_SLOT_COUNT:
return CAPACITY_WAIT
baseline = complete_live_baseline(pool, endpoint)
return (
NONE
if occupied == proven and len(baseline) == proven
else CAPACITY_WAIT
)
def reserve_live_slot(
pool,
endpoint,
ticket,
use=MAIN,
admission_basis=NO_TRANSFER_ADMISSION):
if not transfer_admission_matches(
pool,
endpoint,
ticket,
use,
admission_basis):
return None, SLOT_WAIT
wait = pool_wait_reason(pool, endpoint, ticket)
if wait != NONE:
return None, wait
index = next(
index
for index, slot in enumerate(pool.slots)
if slot.phase == EMPTY
)
pool.last_incarnation += 1
slot = pool.slots[index]
slot.phase = RESERVED
slot.incarnation = pool.last_incarnation
slot.ticket = ticket
slot.attempt = None
slot.use = use
slot.live_since = 0
pool.opening_ticket = ticket
return slot_key(endpoint, index, slot), NONE
def commit_live_slot(
pool,
key,
ticket,
attempt,
use=MAIN,
admission_basis=NO_TRANSFER_ADMISSION):
slot = find_live_slot(pool, key)
if (
slot is None
or slot.phase != RESERVED
or slot.ticket != ticket
or pool.opening_ticket != ticket
or attempt.runtime_id != ticket.runtime_id
or attempt.proxy_generation != ticket.proxy_generation
or not transfer_admission_matches(
pool,
key.endpoint,
ticket,
use,
admission_basis)
):
return False
occupied_before = nonempty_slot_count(pool) - 1
baseline = complete_live_baseline(pool, key.endpoint)
target = pool.proven_lower_bound + 1
continuation_admission = admission_basis == RELEASED_CONTINUATION
expansion = (
not continuation_admission
and
pool.proven_lower_bound > 0
and target <= LIVE_SLOT_COUNT
and occupied_before == pool.proven_lower_bound
and len(baseline) == pool.proven_lower_bound
and pool.probe is None
and (
pool.learned_limit is None
or target <= pool.learned_limit
)
)
slot.phase = OPENING
slot.ticket = None
slot.attempt = attempt
slot.use = use
pool.opening_ticket = None
pool.opening_attempt = attempt
if continuation_admission:
pool.demand_transfer_continuation = None
if expansion:
pool.probe = CapacityProbe(key, attempt, target, baseline)
return True
def mark_live_slot_relay_ready(pool, key, attempt, now):
slot = find_live_slot(pool, key)
if (
slot is None
or slot.phase != OPENING
or slot.attempt != attempt
or pool.opening_attempt != attempt
):
return False
probe_matches = (
pool.probe is not None
and pool.probe.key == key
and pool.probe.attempt == attempt
)
stable_probe = (
probe_matches
and pool.probe.target == pool.proven_lower_bound + 1
and pool.probe.baseline
== complete_live_baseline(pool, key.endpoint)
)
bootstrap = (
pool.proven_lower_bound == 0
and not complete_live_baseline(pool, key.endpoint)
)
slot.phase = LIVE
slot.live_since = now
pool.opening_attempt = None
if stable_probe:
pool.proven_lower_bound = pool.probe.target
elif bootstrap:
pool.proven_lower_bound = 1
if probe_matches:
pool.probe = None
return True
def mark_live_slot_capacity_terminal(
pool,
key,
attempt,
reason,
final_endpoint_terminal):
if not final_endpoint_terminal or reason not in CAPACITY_PRESSURE_REASONS:
return False
slot = find_live_slot(pool, key)
continuation = pool.demand_transfer_continuation
if (
slot is not None
and slot.phase == CLOSING
and slot.attempt == attempt
and continuation is not None
and continuation.stage != CONTINUATION_RELEASED
and continuation.source_key == key
and continuation.source_attempt == attempt
):
pool.demand_transfer_continuation = None
reset_learning_if_empty(pool)
return True
if (
slot is None
or slot.phase != OPENING
or slot.attempt != attempt
or pool.opening_attempt != attempt
):
return False
probe_matches = (
pool.probe is not None
and pool.probe.key == key
and pool.probe.attempt == attempt
)
stable_probe = (
probe_matches
and pool.probe.target == pool.proven_lower_bound + 1
and len(pool.probe.baseline) == pool.proven_lower_bound
and bool(pool.probe.baseline)
and pool.probe.baseline
== complete_live_baseline(pool, key.endpoint)
)
if stable_probe:
pool.learned_limit = len(pool.probe.baseline)
if probe_matches:
pool.probe = None
slot.phase = CLOSING
return True
def begin_live_slot_close(
pool,
key,
attempt,
successor=None,
resume_purpose=None):
slot = find_live_slot(pool, key)
if (
slot is None
or slot.phase not in {OPENING, LIVE}
or slot.attempt != attempt
or (resume_purpose is not None and pool.reclaim is not None)
or (successor is not None and resume_purpose is None)
):
return None
slot.phase = CLOSING
if (
pool.probe is not None
and pool.probe.key == key
and pool.probe.attempt == attempt
):
pool.probe = None
if resume_purpose is not None:
pool.reclaim = LiveReclaim(key, attempt, successor)
return (key, attempt, resume_purpose)
def select_demand_transfer_main_victim(
pool,
successor,
foreground_runtime_id,
main_relay_attempts,
own_runtime):
return next((
(index, slot)
for index, slot in enumerate(pool.slots)
if slot.phase == LIVE
and slot.attempt is not None
and slot.use == MAIN
and slot.attempt.runtime_id != foreground_runtime_id
and (slot.attempt.runtime_id == successor.runtime_id) == own_runtime
and (
not own_runtime
or slot.attempt in main_relay_attempts
)
), None)
def plan_demand_transfer_reservation(
pool,
endpoint,
successor,
use,
foreground_runtime_id,
admission_basis,
main_relay_attempts,
admissible_transfer_waiting):
if (
not endpoint
or use not in {MEDIA, UPLOAD}
or not admissible_transfer_waiting
or not transfer_admission_matches(
pool,
endpoint,
successor,
use,
admission_basis)
):
return WAIT_ACTION, None, SLOT_WAIT
reservation_wait = pool_wait_reason(pool, endpoint, successor)
if pool.reclaim or any(slot.phase == CLOSING for slot in pool.slots):
return WAIT_ACTION, None, CLOSING_WAIT
if pool.opening_ticket or pool.opening_attempt:
return WAIT_ACTION, None, SLOT_WAIT
if admission_basis == RELEASED_CONTINUATION:
return (
RESERVE_ACTION if reservation_wait == NONE else WAIT_ACTION,
None,
reservation_wait,
)
if (
admission_basis != MAIN_RELAY_PROOF
or pool.demand_transfer_continuation is not None
):
return WAIT_ACTION, None, reservation_wait
occupied = nonempty_slot_count(pool)
proven = max(0, min(pool.proven_lower_bound, LIVE_SLOT_COUNT))
bootstrap = not proven and not occupied
replenishment = occupied < proven
baseline = complete_live_baseline(pool, endpoint)
stable_frontier = (
proven > 0
and occupied == proven
and any(slot.phase == EMPTY for slot in pool.slots)
and len(baseline) == proven
)
if reservation_wait == NONE and (bootstrap or replenishment):
return RESERVE_ACTION, None, reservation_wait
if reservation_wait == NONE and not stable_frontier:
return WAIT_ACTION, None, SLOT_WAIT
if reservation_wait not in {NONE, SLOT_WAIT, CAPACITY_WAIT}:
return WAIT_ACTION, None, reservation_wait
victim = select_demand_transfer_main_victim(
pool,
successor,
foreground_runtime_id,
main_relay_attempts,
own_runtime=False)
own_runtime = False
if victim is None:
victim = select_demand_transfer_main_victim(
pool,
successor,
foreground_runtime_id,
main_relay_attempts,
own_runtime=True)
own_runtime = victim is not None
if victim is None:
return (
RESERVE_ACTION if reservation_wait == NONE else WAIT_ACTION,
None,
reservation_wait,
)
index, slot = victim
close = begin_live_slot_close(
pool,
slot_key(endpoint, index, slot),
slot.attempt,
successor,
RECLAIMED_MAIN_RESUME)
if close is None:
wait = SLOT_WAIT if reservation_wait == NONE else reservation_wait
return WAIT_ACTION, None, wait
if own_runtime:
pool.reclaim.continuation_required = True
pool.demand_transfer_continuation = DemandTransferContinuation(
source_key=close[0],
source_attempt=close[1],
successor=successor,
runtime_id=successor.runtime_id,
proxy_generation=successor.proxy_generation,
use=use)
return RECLAIM_ACTION, close, reservation_wait
def cancel_live_slot_successor(pool, ticket):
applied = False
if pool.reclaim is not None and pool.reclaim.successor == ticket:
pool.reclaim.successor = None
applied = True
if (
pool.demand_transfer_continuation is not None
and pool.demand_transfer_continuation.successor == ticket
):
pool.demand_transfer_continuation = None
applied = True
if applied:
reset_learning_if_empty(pool)
return applied
def authorize_live_slot_reclaim(
pool,
key,
attempt,
foreground_runtime_id,
callback_ready=True):
continuation = pool.demand_transfer_continuation
slot = find_live_slot(pool, key)
reclaim = pool.reclaim
if (
slot is None
or slot.phase != CLOSING
or slot.attempt != attempt
or reclaim is None
or reclaim.stage != RECLAIM_REQUESTED
or reclaim.key != key
or reclaim.incumbent != attempt
):
return False, False
continuation_matches = (
continuation is not None
and continuation.stage == CONTINUATION_REQUESTED
and continuation.source_key == key
and continuation.source_attempt == attempt
and reclaim.successor == continuation.successor
)
authorized = (
callback_ready
and attempt.runtime_id != foreground_runtime_id
and reclaim.successor is not None
and (
not reclaim.continuation_required
or continuation_matches
)
and (continuation is None or continuation_matches)
)
if authorized:
reclaim.stage = RECLAIM_AUTHORIZED
if continuation_matches:
continuation.stage = AWAITING_RELEASE
else:
slot.phase = LIVE
if (
continuation_matches
or (
continuation is not None
and continuation.source_key == key
and continuation.source_attempt == attempt
)
):
pool.demand_transfer_continuation = None
pool.reclaim = None
reset_learning_if_empty(pool)
return True, authorized
def release_live_slot(pool, key, attempt):
slot = find_live_slot(pool, key)
if (
slot is None
or slot.phase not in {OPENING, LIVE, CLOSING}
or slot.attempt != attempt
):
return LiveReleaseResult()
if pool.opening_attempt == attempt:
pool.opening_attempt = None
if (
pool.probe is not None
and pool.probe.key == key
and pool.probe.attempt == attempt
):
pool.probe = None
successor = None
continuation_changed = False
reclaim_matches = (
pool.reclaim is not None
and pool.reclaim.key == key
and pool.reclaim.incumbent == attempt
)
authorized_reclaim = (
reclaim_matches
and pool.reclaim.stage == RECLAIM_AUTHORIZED
)
continuation = pool.demand_transfer_continuation
continuation_source_matches = (
continuation is not None
and continuation.stage != CONTINUATION_RELEASED
and continuation.source_key == key
and continuation.source_attempt == attempt
)
continuation_reclaim_matches = (
authorized_reclaim
and continuation_source_matches
and continuation.stage == AWAITING_RELEASE
and pool.reclaim.successor == continuation.successor
)
if reclaim_matches:
if (
continuation_reclaim_matches
and pool.reclaim.continuation_required
):
continuation.stage = CONTINUATION_RELEASED
successor = pool.reclaim.successor
continuation_changed = True
elif continuation_source_matches:
pool.demand_transfer_continuation = None
continuation_changed = True
elif (
authorized_reclaim
and not pool.reclaim.continuation_required
and continuation is None
):
successor = pool.reclaim.successor
pool.reclaim = None
elif continuation_source_matches:
pool.demand_transfer_continuation = None
continuation_changed = True
slot.phase = CLOSING
slot.phase = EMPTY
slot.ticket = None
slot.attempt = None
slot.use = MAIN
slot.live_since = 0
reset_learning_if_empty(pool)
return LiveReleaseResult(True, successor, continuation_changed)
def close_slots_before_generation(
pool,
endpoint,
runtime_id,
proxy_generation,
cancelled_tickets=()):
if pool.opening_ticket in cancelled_tickets:
slot = next(
slot for slot in pool.slots
if slot.phase == RESERVED and slot.ticket == pool.opening_ticket
)
slot.phase = EMPTY
slot.ticket = None
pool.opening_ticket = None
if pool.reclaim and pool.reclaim.successor in cancelled_tickets:
pool.reclaim.successor = None
if (
pool.demand_transfer_continuation
and pool.demand_transfer_continuation.successor in cancelled_tickets
):
pool.demand_transfer_continuation = None
actions = []
for index, slot in enumerate(pool.slots):
if (
slot.phase in {OPENING, LIVE}
and slot.attempt is not None
and slot.attempt.runtime_id == runtime_id
and slot.attempt.proxy_generation < proxy_generation
):
key = slot_key(endpoint, index, slot)
slot.phase = CLOSING
if pool.probe and pool.probe.key == key:
pool.probe = None
actions.append((key, slot.attempt, None))
if (
pool.reclaim
and pool.reclaim.incumbent.runtime_id == runtime_id
and pool.reclaim.incumbent.proxy_generation < proxy_generation
):
pool.reclaim = None
if pool.demand_transfer_continuation:
continuation = pool.demand_transfer_continuation
if (
continuation.runtime_id == runtime_id
and continuation.proxy_generation < proxy_generation
):
pool.demand_transfer_continuation = None
reset_learning_if_empty(pool)
return actions
def close_runtime_live_slots(pool, endpoint, runtime_id):
cancelled = tuple(
slot.ticket
for slot in pool.slots
if slot.phase == RESERVED
and slot.ticket is not None
and slot.ticket.runtime_id == runtime_id
)
return close_slots_before_generation(
pool,
endpoint,
runtime_id,
2 ** 63,
cancelled)
def plan_main_replacement(
pool,
endpoint,
successor,
foreground_runtime_id,
now,
purpose,
admissible_transfer_waiting=False,
transfer_active=False):
if (
(
purpose == FOREGROUND_RECOVERY
and successor.runtime_id != foreground_runtime_id
)
or (
purpose != FOREGROUND_RECOVERY
and successor.runtime_id == foreground_runtime_id
)
or (
purpose == BACKGROUND_DUTY
and (admissible_transfer_waiting or transfer_active)
)
or pool.opening_ticket
or pool.opening_attempt
or pool.reclaim
or pool.demand_transfer_continuation
or any(slot.phase == CLOSING for slot in pool.slots)
):
return None
wait = pool_wait_reason(pool, endpoint, successor)
baseline = complete_live_baseline(pool, endpoint)
occupied = nonempty_slot_count(pool)
if (
wait not in {SLOT_WAIT, CAPACITY_WAIT}
or pool.proven_lower_bound <= 0
or occupied != pool.proven_lower_bound
or len(baseline) != pool.proven_lower_bound
):
return None
candidates = [
(index, slot)
for index, slot in enumerate(pool.slots)
if slot.phase == LIVE
and slot.attempt is not None
and slot.use == MAIN
and slot.attempt.runtime_id != foreground_runtime_id
and slot.attempt.runtime_id != successor.runtime_id
and (
purpose != BACKGROUND_DUTY
or slot.live_since + 60000 <= now
)
]
if not candidates:
return None
index, slot = min(candidates, key=lambda item: item[1].live_since)
return begin_live_slot_close(
pool,
slot_key(endpoint, index, slot),
slot.attempt,
successor,
resume_purpose=RECLAIMED_MAIN_RESUME)
def admission_priority(
ticket,
use,
foreground,
admission_basis,
age):
if use == MAIN and foreground:
return 0
if ticket.purpose == RECLAIMED_MAIN_RESUME:
return 9
if use == MAIN:
return 4 if admission_basis == MAIN_RELAY_PROOF else 3
if (
use in {MEDIA, UPLOAD}
and admission_basis != NO_TRANSFER_ADMISSION
):
return 1 if foreground else 2
base = 5 if use == "maintenance" else 8
return max(4, base - age // 15000)
def select_with_released_continuation(
pool,
candidates,
foreground_runtime_id):
continuation = pool.demand_transfer_continuation
if (
continuation is None
or continuation.stage != CONTINUATION_RELEASED
):
return None
foreground_main = next((
ticket
for ticket, use, current, due in candidates
if use == MAIN
and ticket.runtime_id == foreground_runtime_id
and current
and due
), None)
if foreground_main is not None:
assert cancel_live_slot_successor(pool, continuation.successor)
return foreground_main
return next((
ticket
for ticket, use, current, due in candidates
if ticket == continuation.successor
and use == continuation.use
and current
and due
), None)
def read(path):
assert path.exists(), f"missing expected source file: {path}"
return path.read_text(encoding="utf-8")
def read_endpoint_health_sources():
return "\n".join(read(path) for path in (
ENDPOINT_HEALTH_CPP,
ENDPOINT_HEALTH_LIFECYCLE_CPP,
))
def is_terminal_reason(reason):
return reason != NONE
def is_terminal_failure(status):
return status.error or is_terminal_reason(status.reason)
def is_success(status):
return status.phase == CONNECTED
def relay_success_is_fresh(status, now):
return is_success(status) and status.success_until > now
def is_newer_proxy_epoch(current, update):
if update.proxy_generation != current.proxy_generation:
return update.proxy_generation > current.proxy_generation
if update.proxy_epoch != current.proxy_epoch:
return update.proxy_epoch > current.proxy_epoch
return update.success_epoch > current.success_epoch
def is_newer_attempt(current, update):
if update.proxy_generation != current.proxy_generation:
return update.proxy_generation > current.proxy_generation
if update.proxy_epoch != current.proxy_epoch:
return update.proxy_epoch > current.proxy_epoch
if update.success_epoch != current.success_epoch:
return update.success_epoch > current.success_epoch
return update.attempt_id > current.attempt_id
def is_older_proxy_generation(current, update):
return (
current.proxy_generation
and (not update.proxy_generation
or update.proxy_generation < current.proxy_generation)
)
def is_older_attempt(current, update):
if current.proxy_generation and update.proxy_generation != current.proxy_generation:
return False
if current.proxy_epoch and not update.proxy_epoch:
return True
if update.proxy_epoch and update.proxy_epoch < current.proxy_epoch:
return True
if update.proxy_epoch != current.proxy_epoch:
return False
if current.success_epoch and not update.success_epoch:
return True
if update.success_epoch and update.success_epoch < current.success_epoch:
return True
if update.success_epoch != current.success_epoch:
return False
return (
current.attempt_id
and update.attempt_id
and update.attempt_id < current.attempt_id
)
def is_relay_data_stall(status):
return status.reason in {
NO_APPDATA,
NO_MTPROTO,
MTP_TIMEOUT_AFTER_DATA,
}
def normalize_terminal_reason(current, status):
if status.reason != NO_SERVERHELLO or current.attempt != status.attempt:
return status
if current.phase not in {CHECKING, CONNECTED}:
return status
return replace(status, reason=NO_APPDATA, error=False)
def apply_selected_status_update(current, update, now):
if is_older_proxy_generation(current.attempt, update.attempt):
return current
if is_older_attempt(current.attempt, update.attempt):
return current
if (
relay_success_is_fresh(current, now)
and is_terminal_failure(update)
and update.attempt != current.attempt
and not is_newer_proxy_epoch(current.attempt, update.attempt)
):
return current
if not is_terminal_reason(current.reason):
return update
if (
is_success(update)
or is_terminal_reason(update.reason)
or is_newer_attempt(current.attempt, update.attempt)
):
return update
if current.terminal_until > now:
return current
return update
def reduce_status(current, fact, now=1000, fresh_window=15000):
status = fact.status
if status.attempt.probe:
return current
if is_relay_data_stall(status):
status = replace(status, error=False)
if fact.success_scope == RELAY:
status = replace(
status,
phase=CONNECTED,
success_until=now + fresh_window)
status = normalize_terminal_reason(current, status)
if (
relay_success_is_fresh(current, now)
and is_terminal_failure(status)
and status.attempt != current.attempt
and not is_newer_proxy_epoch(current.attempt, status.attempt)
):
return current
return apply_selected_status_update(current, status, now)
def report_epoch_is_stale(report_epoch, state):
return report_epoch and report_epoch < state.proxy_epoch
def report_success_epoch_is_stale(report_success_epoch, state):
return state.success_epoch and report_success_epoch < state.success_epoch
def relay_proof_identity(report):
return RelayProofIdentity(
runtime_id=report.runtime_id,
proxy_generation=report.proxy_generation,
attempt_id=report.attempt_id)
def runtime_proxy_generation_is_stale(state, runtime_id, proxy_generation):
return (
runtime_id in state.generations
and proxy_generation < state.generations[runtime_id]
)
def has_endpoint_attempt(state, identity):
admission = state.attempt_starts.get(identity.attempt_id)
return (
admission is not None
and admission.runtime_id == identity.runtime_id
and admission.proxy_generation == identity.proxy_generation
)
def has_relay_proof(state, identity):
return identity in state.relay_proofs
def synchronize_relay_proof_aggregate(state):
state.relay_proofs = {
identity: proof
for identity, proof in state.relay_proofs.items()
if not (
identity.runtime_id in state.generations
and identity.proxy_generation
< state.generations[identity.runtime_id]
)
}
state.relay_proven = bool(state.relay_proofs)
state.last_relay_success_at = max(
(proof.proven_at for proof in state.relay_proofs.values()),
default=0)
if state.relay_proven:
state.healthy = True
def apply_runtime_proxy_generation(state, runtime_id, proxy_generation):
if not runtime_id:
return
current = state.generations.get(runtime_id)
if (
(current is not None and proxy_generation <= current)
or (current is None and not proxy_generation)
):
return
state.generations[runtime_id] = proxy_generation
state.attempt_starts = {
attempt_id: admission
for attempt_id, admission in state.attempt_starts.items()
if not (
admission.runtime_id == runtime_id
and admission.proxy_generation < proxy_generation
)
}
state.relay_proofs = {
identity: proof
for identity, proof in state.relay_proofs.items()
if not (
identity.runtime_id == runtime_id
and identity.proxy_generation < proxy_generation
)
}
synchronize_relay_proof_aggregate(state)
def prune_expired_endpoint_state(state, now):
state.attempt_starts = {
attempt_id: admission
for attempt_id, admission in state.attempt_starts.items()
if now - admission.started_at <= ATTEMPT_HARD_TTL
}
def seed_admission(store, report):
state = store.state(report.endpoint)
apply_runtime_proxy_generation(
state,
report.runtime_id,
report.proxy_generation)
state.attempt_starts[report.attempt_id] = EndpointAttemptState(
runtime_id=report.runtime_id,
proxy_generation=report.proxy_generation,
started_at=report.started_at)
def promote_relay_proof(state, identity, proof):
if has_relay_proof(state, identity):
return ALREADY_PROVEN
if not has_endpoint_attempt(state, identity):
return MISSING_ADMISSION
state.relay_proofs[identity] = proof
del state.attempt_starts[identity.attempt_id]
synchronize_relay_proof_aggregate(state)
return INSERTED
def retire_relay_proof_record(state, identity):
if identity not in state.relay_proofs:
return False
del state.relay_proofs[identity]
synchronize_relay_proof_aggregate(state)
return True
def failure_from_stale_attempt(report, state):
if runtime_proxy_generation_is_stale(
state,
report.runtime_id,
report.proxy_generation):
return True
if has_relay_proof(state, relay_proof_identity(report)):
return False
return (
report_epoch_is_stale(report.proxy_epoch, state)
or report_success_epoch_is_stale(report.success_epoch, state)
or (
state.last_relay_success_at
and report.started_at < state.last_relay_success_at
)
)
def success_from_stale_attempt(report, state):
if runtime_proxy_generation_is_stale(
state,
report.runtime_id,
report.proxy_generation):
return True
identity = relay_proof_identity(report)
if has_endpoint_attempt(state, identity) or has_relay_proof(state, identity):
return False
return (
report_epoch_is_stale(report.proxy_epoch, state)
or report_success_epoch_is_stale(report.success_epoch, state)
or (
state.last_relay_success_at
and report.started_at < state.last_relay_success_at
)
)
def report_relay_success(store, report, now):
state = store.state(report.endpoint)
prune_expired_endpoint_state(state, now)
if runtime_proxy_generation_is_stale(
state,
report.runtime_id,
report.proxy_generation):
return STALE_GENERATION
apply_runtime_proxy_generation(
state,
report.runtime_id,
report.proxy_generation)
if success_from_stale_attempt(report, state):
return STALE_ATTEMPT
promotion = promote_relay_proof(
state,
relay_proof_identity(report),
RelayProofState(proven_at=now))
if promotion != INSERTED:
return promotion
state.last_success_at = now
state.route_state = "good"
state.route_success_count += 1
state.success_epoch += 1
state.proxy_epoch += 1
state.capability_success_count += 1
state.capability_relay_proven = True
state.last_failure = NONE
state.cooldown_until = 0
state.healthy = True
return INSERTED
def faketls_appdata_success(store, report, now):
state = store.state(report.endpoint)
prune_expired_endpoint_state(state, now)
if runtime_proxy_generation_is_stale(
state,
report.runtime_id,
report.proxy_generation):
return False
apply_runtime_proxy_generation(
state,
report.runtime_id,
report.proxy_generation)
if success_from_stale_attempt(report, state):
return False
state.last_success_at = now
return True
def retire_relay_proof(store, report, now):
state = store.states.get(report.endpoint)
if state is None:
return MISSING_OR_DUPLICATE
prune_expired_endpoint_state(state, now)
if runtime_proxy_generation_is_stale(
state,
report.runtime_id,
report.proxy_generation):
return STALE_GENERATION
apply_runtime_proxy_generation(
state,
report.runtime_id,
report.proxy_generation)
if not retire_relay_proof_record(state, relay_proof_identity(report)):
return MISSING_OR_DUPLICATE
return (
RETIRED_WITH_SURVIVORS
if state.relay_proven
else RETIRED_FINAL
)
def punish_final_failure(state, reason, now):
state.route_state = "bad"
state.route_failure_count += 1
state.last_failure = reason
relay_failures = {
NO_APPDATA,
NO_MTPROTO,
MTP_TIMEOUT_AFTER_DATA,
}
cooldown_failures = {
NO_SERVERHELLO,
NO_APPDATA,
NO_MTPROTO,
HMAC_MISMATCH,
}
if reason not in relay_failures:
state.capability_failure_count += 1
if reason in cooldown_failures:
state.cooldown_count += 1
state.cooldown_until = now + 15000
state.rotation_count += 1
state.healthy = False
if reason in relay_failures:
state.relay_capability_failure_count += 1
state.capability_relay_proven = False
def report_failure(store, report, reason, now):
state = store.state(report.endpoint)
prune_expired_endpoint_state(state, now)
if runtime_proxy_generation_is_stale(
state,
report.runtime_id,
report.proxy_generation):
return STALE_GENERATION
apply_runtime_proxy_generation(
state,
report.runtime_id,
report.proxy_generation)
if failure_from_stale_attempt(report, state):
return STALE_ATTEMPT
identity = relay_proof_identity(report)
owned_attempt = has_endpoint_attempt(state, identity)
retired = retire_relay_proof_record(state, identity)
if not owned_attempt and not retired:
return MISSING_OR_DUPLICATE
if owned_attempt:
del state.attempt_starts[report.attempt_id]
if reason == NO_SERVERHELLO:
retry_until = now + OPENING_PRESSURE_COOLDOWN
if retry_until > state.opening_pressure_retry_until:
state.opening_pressure_reason = reason
state.opening_pressure_retry_until = retry_until
if state.relay_proven:
return (
RETIRED_WITH_SURVIVORS
if retired
else TERMINAL_ATTEMPT
)
punish_final_failure(state, reason, now)
return RETIRED_FINAL if retired else TERMINAL_ATTEMPT
def report_relay_stall(store, report, now):
result = retire_relay_proof(store, report, now)
if result == RETIRED_FINAL:
state = store.states[report.endpoint]
state.relay_capability_failure_count += 1
state.capability_relay_proven = False
return result
def neutral_retire(store, report, now, origin=NONE):
assert origin in {
NONE,
"normal_close",
"cancel",
"owner_destruction",
"proxy_switch",
}
return retire_relay_proof(store, report, now)
def healthy_remote_close(store, report, now):
state = store.states.get(report.endpoint)
if state is None or not state.healthy:
return MISSING_OR_DUPLICATE
return neutral_retire(
store,
report,
now,
origin="normal_close")
def snapshot_endpoint_state(store, endpoint, now):
state = store.states.get(endpoint)
if state is None:
return None
prune_expired_endpoint_state(state, now)
return deepcopy(state)
def apply_proxy_generation(store, runtime_id, proxy_generation, now):
for state in store.states.values():
prune_expired_endpoint_state(state, now)
apply_runtime_proxy_generation(
state,
runtime_id,
proxy_generation)
def unregister_runtime(store, runtime_id):
for state in store.states.values():
state.generations.pop(runtime_id, None)
state.attempt_starts = {
attempt_id: admission
for attempt_id, admission in state.attempt_starts.items()
if admission.runtime_id != runtime_id
}
state.relay_proofs = {
identity: proof
for identity, proof in state.relay_proofs.items()
if identity.runtime_id != runtime_id
}
synchronize_relay_proof_aggregate(state)
def punishment_signature(state):
return (
state.route_state,
state.route_success_count,
state.route_failure_count,
state.cooldown_until,
state.cooldown_count,
state.rotation_count,
state.capability_success_count,
state.capability_failure_count,
state.relay_capability_failure_count,
state.capability_relay_proven,
)
def broker_claim_front(state):
if (
not state.active
or not state.has_context
or state.admission
or state.admission_in_progress
or state.start_scheduled
):
return False
state.admission_in_progress = True
return True
def broker_complete_start(state):
state.admission_in_progress = False
state.start_scheduled = True
def broker_cancel(state):
state.active = False
state.admission = False
state.admission_lease_active = False
state.admission_in_progress = False
def probe_decision(active_probe_key, snapshot, now=1000, active_window=9000):
if active_probe_key:
return "waiting_for_connection_slot"
if (
snapshot["healthy"]
and not snapshot["half_open"]
and snapshot["relay_proven"]
and snapshot["last_relay_success_at"]
and now - snapshot["last_relay_success_at"] < active_window
):
return "connected_by_active_session"
return "start_probe"
def capability_relay_proven_after_failure(relay_proven, reason, degraded):
if degraded and reason in {NO_APPDATA, NO_MTPROTO, MTP_TIMEOUT_AFTER_DATA}:
return False
return relay_proven
def capability_relay_proven_after_age(relay_proven, proven_at, now, ttl):
return relay_proven and proven_at and now - proven_at <= ttl
def live_ticket(
ticket_id,
runtime_id,
proxy_generation=1,
purpose=ORDINARY):
return LiveTicket(
runtime_id=runtime_id,
ticket_id=ticket_id,
revision=ticket_id + 100,
proxy_generation=proxy_generation,
purpose=purpose)
def live_attempt(ticket, attempt_id):
return RelayProofIdentity(
runtime_id=ticket.runtime_id,
proxy_generation=ticket.proxy_generation,
attempt_id=attempt_id)
def open_pool_attempt(
pool,
ticket,
attempt_id,
use=MAIN,
endpoint=ABC_ENDPOINT):
admission_basis = (
MAIN_RELAY_PROOF
if use in {MEDIA, UPLOAD}
else NO_TRANSFER_ADMISSION
)
key, wait = reserve_live_slot(
pool,
endpoint,
ticket,
use,
admission_basis)
assert wait == NONE
assert key is not None
attempt = live_attempt(ticket, attempt_id)
assert commit_live_slot(
pool,
key,
ticket,
attempt,
use,
admission_basis)
return key, attempt
def add_live_pool_attempt(
pool,
ticket,
attempt_id,
use=MAIN,
now=1000,
endpoint=ABC_ENDPOINT):
key, attempt = open_pool_attempt(
pool,
ticket,
attempt_id,
use,
endpoint)
assert mark_live_slot_relay_ready(pool, key, attempt, now)
return key, attempt
def one_live_plus_probe():
pool = LivePool()
first = live_ticket(1, 10)
first_key, first_attempt = add_live_pool_attempt(
pool,
first,
101,
now=100)
second = live_ticket(2, 20)
second_key, second_attempt = open_pool_attempt(
pool,
second,
102)
assert pool.probe == CapacityProbe(
second_key,
second_attempt,
2,
(first_key,))
return pool, first_key, first_attempt, second_key, second_attempt
def test_live_pool_bootstrap_and_sequential_expansion():
pool = LivePool()
live = []
for index in range(LIVE_SLOT_COUNT):
ticket = live_ticket(index + 1, index + 10)
key, attempt = open_pool_attempt(pool, ticket, index + 100)
if index == 0:
assert pool.probe is None
else:
assert pool.probe is not None
assert pool.probe.target == index + 1
assert pool.probe.baseline == tuple(key for key, _ in live)
blocked = live_ticket(index + 20, index + 30)
assert reserve_live_slot(pool, ABC_ENDPOINT, blocked) == (
None,
SLOT_WAIT)
assert mark_live_slot_relay_ready(
pool,
key,
attempt,
1000 + index)
assert pool.proven_lower_bound == index + 1
assert pool.opening_attempt is None
assert pool.probe is None
live.append((key, attempt))
assert len(pool.slots) == LIVE_SLOT_COUNT
assert all(slot.phase == LIVE for slot in pool.slots)
assert reserve_live_slot(
pool,
ABC_ENDPOINT,
live_ticket(99, 99)) == (None, SLOT_WAIT)
def test_capacity_learning_accepts_only_exact_stable_frontier_evidence():
for reason in sorted(CAPACITY_PRESSURE_REASONS):
pool, _, _, key, attempt = one_live_plus_probe()
assert mark_live_slot_capacity_terminal(
pool,
key,
attempt,
reason,
True)
assert pool.slots[key.index].phase == CLOSING
assert pool.learned_limit == 1
assert pool.opening_attempt == attempt
pool, _, _, key, attempt = one_live_plus_probe()
assert not mark_live_slot_capacity_terminal(
pool,
key,
attempt,
TCP_TIMEOUT,
False)
assert pool.learned_limit is None
assert pool.slots[key.index].phase == OPENING
for reason in (DNS_FAILED, CANCELLED, REMOTE_CLOSED, HMAC_MISMATCH):
pool, _, _, key, attempt = one_live_plus_probe()
assert not mark_live_slot_capacity_terminal(
pool,
key,
attempt,
reason,
True)
assert pool.learned_limit is None
pool, _, _, key, attempt = one_live_plus_probe()
stale_key = replace(key, incarnation=key.incarnation + 1)
stale_attempt = replace(attempt, attempt_id=attempt.attempt_id + 1)
assert not mark_live_slot_capacity_terminal(
pool,
stale_key,
attempt,
TCP_TIMEOUT,
True)
assert not mark_live_slot_capacity_terminal(
pool,
key,
stale_attempt,
TCP_TIMEOUT,
True)
assert pool.learned_limit is None
pool, baseline_key, baseline_attempt, key, attempt = one_live_plus_probe()
assert begin_live_slot_close(pool, baseline_key, baseline_attempt)
assert mark_live_slot_capacity_terminal(
pool,
key,
attempt,
TCP_TIMEOUT,
True)
assert pool.learned_limit is None
pool = LivePool()
key, attempt = open_pool_attempt(pool, live_ticket(1, 1), 1)
assert pool.probe is None
assert mark_live_slot_capacity_terminal(
pool,
key,
attempt,
TCP_TIMEOUT,
True)
assert pool.learned_limit is None
assert not mark_live_slot_capacity_terminal(
pool,
key,
attempt,
TCP_TIMEOUT,
True)
pool, _, _, key, attempt = one_live_plus_probe()
pool.probe = None
assert mark_live_slot_capacity_terminal(
pool,
key,
attempt,
TCP_TIMEOUT,
True)
assert pool.learned_limit is None
pool, _, _, key, attempt = one_live_plus_probe()
cleanup = close_slots_before_generation(
pool,
ABC_ENDPOINT,
attempt.runtime_id,
attempt.proxy_generation + 1)
assert cleanup == [(key, attempt, None)]
assert not mark_live_slot_capacity_terminal(
pool,
key,
attempt,
TCP_TIMEOUT,
True)
assert pool.learned_limit is None
def test_learned_cap_and_reclaim_are_serialized_until_exact_release():
pool, _, _, probe_key, probe_attempt = one_live_plus_probe()
assert mark_live_slot_capacity_terminal(
pool,
probe_key,
probe_attempt,
TCP_TIMEOUT,
True)
assert release_live_slot(pool, probe_key, probe_attempt) == (
LiveReleaseResult(slot_released=True))
assert pool.learned_limit == 1
successor = live_ticket(3, 10)
assert reserve_live_slot(pool, ABC_ENDPOINT, successor) == (
None,
CAPACITY_WAIT)
incumbent_key = next(
slot_key(ABC_ENDPOINT, index, slot)
for index, slot in enumerate(pool.slots)
if slot.phase == LIVE)
incumbent = pool.slots[incumbent_key.index].attempt
decision, action, wait = plan_demand_transfer_reservation(
pool,
ABC_ENDPOINT,
successor,
MEDIA,
foreground_runtime_id=99,
admission_basis=MAIN_RELAY_PROOF,
main_relay_attempts=(incumbent,),
admissible_transfer_waiting=True)
assert decision == RECLAIM_ACTION
assert wait == CAPACITY_WAIT
assert action == (
incumbent_key,
incumbent,
RECLAIMED_MAIN_RESUME)
assert pool.reclaim == LiveReclaim(
incumbent_key,
incumbent,
successor,
continuation_required=True)
assert pool.demand_transfer_continuation == DemandTransferContinuation(
source_key=incumbent_key,
source_attempt=incumbent,
successor=successor,
runtime_id=successor.runtime_id,
proxy_generation=successor.proxy_generation,
use=MEDIA)
assert reserve_live_slot(
pool,
ABC_ENDPOINT,
successor,
MEDIA,
MAIN_RELAY_PROOF) == (
None,
CLOSING_WAIT)
second_decision, second_action, second_wait = (
plan_demand_transfer_reservation(
pool,
ABC_ENDPOINT,
live_ticket(4, 10),
MEDIA,
foreground_runtime_id=99,
admission_basis=MAIN_RELAY_PROOF,
main_relay_attempts=(incumbent,),
admissible_transfer_waiting=True))
assert second_decision == WAIT_ACTION
assert second_action is None
assert second_wait == CLOSING_WAIT
assert cancel_live_slot_successor(pool, successor)
assert pool.reclaim.successor is None
assert pool.demand_transfer_continuation is None
assert pool.slots[incumbent_key.index].phase == CLOSING
assert release_live_slot(pool, incumbent_key, incumbent) == (
LiveReleaseResult(slot_released=True))
assert pool.proven_lower_bound == 0
assert pool.learned_limit is None
old_incarnation = incumbent_key.incarnation
new_key, _ = reserve_live_slot(pool, ABC_ENDPOINT, live_ticket(5, 50))
assert new_key.incarnation > old_incarnation
pinned_pool, _, _, probe_key, probe_attempt = one_live_plus_probe()
assert mark_live_slot_capacity_terminal(
pinned_pool,
probe_key,
probe_attempt,
TCP_TIMEOUT,
True)
assert release_live_slot(
pinned_pool,
probe_key,
probe_attempt) == LiveReleaseResult(slot_released=True)
incumbent_key = next(
slot_key(ABC_ENDPOINT, index, slot)
for index, slot in enumerate(pinned_pool.slots)
if slot.phase == LIVE)
incumbent = pinned_pool.slots[incumbent_key.index].attempt
successor = live_ticket(6, 10)
decision, action, _ = plan_demand_transfer_reservation(
pinned_pool,
ABC_ENDPOINT,
successor,
MEDIA,
foreground_runtime_id=99,
admission_basis=MAIN_RELAY_PROOF,
main_relay_attempts=(incumbent,),
admissible_transfer_waiting=True)
assert decision == RECLAIM_ACTION
assert action is not None
assert pinned_pool.demand_transfer_continuation.stage == (
CONTINUATION_REQUESTED)
assert authorize_live_slot_reclaim(
pinned_pool,
incumbent_key,
incumbent,
foreground_runtime_id=99) == (True, True)
assert pinned_pool.demand_transfer_continuation.stage == AWAITING_RELEASE
assert release_live_slot(
pinned_pool,
incumbent_key,
incumbent) == LiveReleaseResult(
slot_released=True,
successor=successor,
continuation_changed=True)
assert pinned_pool.demand_transfer_continuation.stage == (
CONTINUATION_RELEASED)
successor_key, successor_wait = reserve_live_slot(
pinned_pool,
ABC_ENDPOINT,
successor,
MEDIA,
RELEASED_CONTINUATION)
assert successor_wait == NONE
successor_attempt = live_attempt(successor, 600)
assert commit_live_slot(
pinned_pool,
successor_key,
successor,
successor_attempt,
MEDIA,
RELEASED_CONTINUATION)
assert pinned_pool.demand_transfer_continuation is None
assert pinned_pool.probe is None
def test_demand_owned_continuation_survives_exact_main_proof_retirement():
pool = LivePool()
foreground_key, foreground_attempt = add_live_pool_attempt(
pool,
live_ticket(1, 1),
101,
MAIN)
demand_main_key, demand_main_attempt = add_live_pool_attempt(
pool,
live_ticket(2, 2),
102,
MAIN)
pool.learned_limit = 2
successor = live_ticket(3, 2)
decision, close, wait = plan_demand_transfer_reservation(
pool,
ABC_ENDPOINT,
successor,
MEDIA,
foreground_runtime_id=1,
admission_basis=MAIN_RELAY_PROOF,
main_relay_attempts=(demand_main_attempt,),
admissible_transfer_waiting=True)
assert decision == RECLAIM_ACTION
assert wait == CAPACITY_WAIT
assert close == (
demand_main_key,
demand_main_attempt,
RECLAIMED_MAIN_RESUME)
assert pool.slots[foreground_key.index].phase == LIVE
assert pool.slots[foreground_key.index].attempt == foreground_attempt
assert pool.demand_transfer_continuation.stage == CONTINUATION_REQUESTED
terminal_pool = deepcopy(pool)
assert mark_live_slot_capacity_terminal(
terminal_pool,
demand_main_key,
demand_main_attempt,
TCP_TIMEOUT,
True)
assert terminal_pool.demand_transfer_continuation is None
assert terminal_pool.slots[demand_main_key.index].phase == CLOSING
generation_pool = deepcopy(pool)
assert close_slots_before_generation(
generation_pool,
ABC_ENDPOINT,
runtime_id=2,
proxy_generation=2) == []
assert generation_pool.demand_transfer_continuation is None
assert generation_pool.slots[demand_main_key.index].phase == CLOSING
missing_pool = deepcopy(pool)
assert authorize_live_slot_reclaim(
missing_pool,
demand_main_key,
demand_main_attempt,
foreground_runtime_id=1,
callback_ready=False) == (True, False)
assert missing_pool.demand_transfer_continuation is None
assert missing_pool.slots[demand_main_key.index].phase == LIVE
assert missing_pool.reclaim is None
unacknowledged_pool = deepcopy(pool)
assert release_live_slot(
unacknowledged_pool,
demand_main_key,
demand_main_attempt) == LiveReleaseResult(
slot_released=True,
continuation_changed=True)
assert unacknowledged_pool.demand_transfer_continuation is None
assert reserve_live_slot(
pool,
ABC_ENDPOINT,
successor,
MEDIA,
NO_TRANSFER_ADMISSION) == (None, SLOT_WAIT)
assert release_live_slot(
pool,
replace(demand_main_key, incarnation=demand_main_key.incarnation + 1),
demand_main_attempt) == LiveReleaseResult()
assert pool.demand_transfer_continuation.stage == CONTINUATION_REQUESTED
assert authorize_live_slot_reclaim(
pool,
replace(demand_main_key, incarnation=demand_main_key.incarnation + 1),
demand_main_attempt,
foreground_runtime_id=1) == (False, False)
assert pool.demand_transfer_continuation.stage == CONTINUATION_REQUESTED
assert authorize_live_slot_reclaim(
pool,
demand_main_key,
demand_main_attempt,
foreground_runtime_id=1) == (True, True)
assert pool.demand_transfer_continuation.stage == AWAITING_RELEASE
assert authorize_live_slot_reclaim(
pool,
demand_main_key,
demand_main_attempt,
foreground_runtime_id=1) == (False, False)
assert pool.demand_transfer_continuation.stage == AWAITING_RELEASE
release = release_live_slot(
pool,
demand_main_key,
demand_main_attempt)
assert release == LiveReleaseResult(
slot_released=True,
successor=successor,
continuation_changed=True)
assert pool.demand_transfer_continuation.stage == CONTINUATION_RELEASED
assert authorize_live_slot_reclaim(
pool,
demand_main_key,
demand_main_attempt,
foreground_runtime_id=1) == (False, False)
assert release_live_slot(
pool,
demand_main_key,
demand_main_attempt) == LiveReleaseResult()
assert pool.demand_transfer_continuation.stage == CONTINUATION_RELEASED
for stale, use in (
(replace(successor, revision=successor.revision + 1), MEDIA),
(replace(successor, proxy_generation=2), MEDIA),
(replace(successor, runtime_id=3), MEDIA),
(successor, UPLOAD)):
assert reserve_live_slot(
pool,
ABC_ENDPOINT,
stale,
use,
RELEASED_CONTINUATION) == (None, SLOT_WAIT)
assert pool.demand_transfer_continuation.stage == (
CONTINUATION_RELEASED)
key, wait = reserve_live_slot(
pool,
ABC_ENDPOINT,
successor,
MEDIA,
RELEASED_CONTINUATION)
assert wait == NONE
attempt = live_attempt(successor, 103)
assert not commit_live_slot(
pool,
key,
replace(successor, revision=successor.revision + 1),
attempt,
MEDIA,
RELEASED_CONTINUATION)
assert pool.demand_transfer_continuation.stage == CONTINUATION_RELEASED
assert commit_live_slot(
pool,
key,
successor,
attempt,
MEDIA,
RELEASED_CONTINUATION)
assert pool.demand_transfer_continuation is None
assert pool.probe is None
def test_reclaim_victim_order_and_foreground_main_protection():
trace_pool = LivePool()
foreground_key, foreground_attempt = add_live_pool_attempt(
trace_pool,
live_ticket(11, 1),
11,
MAIN)
demand_main_key, demand_main_attempt = add_live_pool_attempt(
trace_pool,
live_ticket(12, 2),
12,
MAIN)
foreign_key, foreign_attempt = add_live_pool_attempt(
trace_pool,
live_ticket(17, 3),
17,
MAIN)
transfer_key, transfer_attempt = add_live_pool_attempt(
trace_pool,
live_ticket(18, 4),
18,
MEDIA)
assert trace_pool.proven_lower_bound == 4
assert trace_pool.probe is None
assert trace_pool.opening_ticket is None
assert trace_pool.opening_attempt is None
successor = live_ticket(19, 2)
decision, action, wait = plan_demand_transfer_reservation(
trace_pool,
ABC_ENDPOINT,
successor,
MEDIA,
foreground_runtime_id=1,
admission_basis=MAIN_RELAY_PROOF,
main_relay_attempts=(demand_main_attempt,),
admissible_transfer_waiting=True)
assert decision == RECLAIM_ACTION
assert wait == SLOT_WAIT
assert action == (
foreign_key,
foreign_attempt,
RECLAIMED_MAIN_RESUME)
assert trace_pool.slots[foreign_key.index].phase == CLOSING
assert trace_pool.reclaim == LiveReclaim(
foreign_key,
foreign_attempt,
successor)
assert trace_pool.demand_transfer_continuation is None
assert trace_pool.slots[foreground_key.index].phase == LIVE
assert trace_pool.slots[foreground_key.index].attempt == foreground_attempt
assert trace_pool.slots[demand_main_key.index].phase == LIVE
assert trace_pool.slots[transfer_key.index].phase == LIVE
assert trace_pool.slots[transfer_key.index].attempt == transfer_attempt
assert trace_pool.opening_ticket is None
assert trace_pool.opening_attempt is None
assert trace_pool.probe is None
stale_revision = replace(successor, revision=successor.revision + 1)
assert not cancel_live_slot_successor(trace_pool, stale_revision)
assert trace_pool.reclaim.successor == successor
assert trace_pool.slots[foreign_key.index].phase == CLOSING
assert reserve_live_slot(
trace_pool,
ABC_ENDPOINT,
successor,
MEDIA,
MAIN_RELAY_PROOF) == (
None,
CLOSING_WAIT)
preauthorization = deepcopy(trace_pool)
assert release_live_slot(
preauthorization,
foreign_key,
foreign_attempt) == LiveReleaseResult(slot_released=True)
assert authorize_live_slot_reclaim(
trace_pool,
foreign_key,
foreign_attempt,
foreground_runtime_id=1) == (True, True)
assert release_live_slot(
trace_pool,
foreign_key,
foreign_attempt) == LiveReleaseResult(
slot_released=True,
successor=successor)
successor_key, successor_wait = reserve_live_slot(
trace_pool,
ABC_ENDPOINT,
successor,
MEDIA,
MAIN_RELAY_PROOF)
assert successor_wait == NONE
successor_attempt = live_attempt(successor, 19)
assert commit_live_slot(
trace_pool,
successor_key,
successor,
successor_attempt,
MEDIA,
MAIN_RELAY_PROOF)
assert trace_pool.probe is None
expansion_pool = LivePool()
for ticket_id, use in ((21, MAIN), (22, MEDIA), (23, UPLOAD)):
add_live_pool_attempt(
expansion_pool,
live_ticket(ticket_id, 2),
ticket_id,
use)
trace28 = live_ticket(28, 2)
decision, action, wait = plan_demand_transfer_reservation(
expansion_pool,
ABC_ENDPOINT,
trace28,
MEDIA,
foreground_runtime_id=2,
admission_basis=MAIN_RELAY_PROOF,
main_relay_attempts=(),
admissible_transfer_waiting=True)
assert decision == RESERVE_ACTION
assert action is None
assert wait == NONE
trace28_key, trace28_wait = reserve_live_slot(
expansion_pool,
ABC_ENDPOINT,
trace28,
MEDIA,
MAIN_RELAY_PROOF)
assert trace28_wait == NONE
trace28_attempt = live_attempt(trace28, 28)
assert commit_live_slot(
expansion_pool,
trace28_key,
trace28,
trace28_attempt,
MEDIA,
MAIN_RELAY_PROOF)
assert expansion_pool.probe == CapacityProbe(
trace28_key,
trace28_attempt,
4,
complete_live_baseline(expansion_pool, ABC_ENDPOINT))
protected = LivePool()
key, attempt = add_live_pool_attempt(
protected,
live_ticket(4, 40),
104,
MAIN)
protected.learned_limit = 1
decision, action, wait = plan_demand_transfer_reservation(
protected,
ABC_ENDPOINT,
live_ticket(5, 40),
MEDIA,
foreground_runtime_id=40,
admission_basis=MAIN_RELAY_PROOF,
main_relay_attempts=(attempt,),
admissible_transfer_waiting=True)
assert decision == WAIT_ACTION
assert action is None
assert wait == CAPACITY_WAIT
assert protected.slots[key.index].phase == LIVE
assert protected.slots[key.index].attempt == attempt
def test_cleanup_is_no_resume_closing_and_late_events_are_incarnation_safe():
pool = LivePool()
old_key, old_attempt = add_live_pool_attempt(
pool,
live_ticket(1, 10, proxy_generation=1),
101,
now=100)
survivor_key, survivor_attempt = add_live_pool_attempt(
pool,
live_ticket(2, 20, proxy_generation=1),
102,
now=200)
pool.learned_limit = 2
last_incarnation = pool.last_incarnation
actions = close_slots_before_generation(
pool,
ABC_ENDPOINT,
runtime_id=10,
proxy_generation=2)
assert actions == [(old_key, old_attempt, None)]
assert pool.slots[old_key.index].phase == CLOSING
assert pool.slots[survivor_key.index].phase == LIVE
assert pool.proven_lower_bound == 2
assert pool.learned_limit == 2
assert not mark_live_slot_relay_ready(pool, old_key, old_attempt, 300)
assert not mark_live_slot_capacity_terminal(
pool,
old_key,
old_attempt,
TCP_TIMEOUT,
True)
assert release_live_slot(
pool,
replace(old_key, incarnation=old_key.incarnation + 1),
old_attempt) == LiveReleaseResult()
assert release_live_slot(
pool,
old_key,
replace(old_attempt, attempt_id=999)) == LiveReleaseResult()
assert release_live_slot(
pool,
old_key,
old_attempt) == LiveReleaseResult(slot_released=True)
assert pool.proven_lower_bound == 2
assert pool.learned_limit == 2
assert pool.last_incarnation == last_incarnation
runtime_actions = close_runtime_live_slots(
pool,
ABC_ENDPOINT,
survivor_attempt.runtime_id)
assert runtime_actions == [(survivor_key, survivor_attempt, None)]
assert pool.slots[survivor_key.index].phase == CLOSING
assert release_live_slot(
pool,
survivor_key,
survivor_attempt) == LiveReleaseResult(slot_released=True)
assert pool.proven_lower_bound == 0
assert pool.learned_limit is None
assert pool.last_incarnation == last_incarnation
def test_reclaimed_resume_tracks_current_foreground_and_background_priority():
resumed = live_ticket(
1,
10,
purpose=RECLAIMED_MAIN_RESUME)
for age in (0, 15000, 60000, 600000):
assert admission_priority(
resumed,
MAIN,
foreground=True,
admission_basis=NO_TRANSFER_ADMISSION,
age=age) == 0
assert admission_priority(
resumed,
MAIN,
foreground=False,
admission_basis=NO_TRANSFER_ADMISSION,
age=age) == 9
ordinary = live_ticket(2, 20)
assert admission_priority(
ordinary,
MAIN,
foreground=False,
admission_basis=MAIN_RELAY_PROOF,
age=0) < 9
demanded = live_ticket(10, 20)
assert admission_priority(
demanded,
MEDIA,
foreground=False,
admission_basis=MAIN_RELAY_PROOF,
age=0) == 2
assert admission_priority(
demanded,
MEDIA,
foreground=True,
admission_basis=MAIN_RELAY_PROOF,
age=0) == 1
assert admission_priority(
ordinary,
MAIN,
foreground=False,
admission_basis=NO_TRANSFER_ADMISSION,
age=0) == 3
pool = LivePool()
oldest_key, oldest_attempt = add_live_pool_attempt(
pool,
live_ticket(3, 30),
103,
MAIN,
now=10)
newer_key, _ = add_live_pool_attempt(
pool,
live_ticket(4, 40),
104,
MAIN,
now=100)
pool.learned_limit = 2
successor = live_ticket(5, 50)
action = plan_main_replacement(
pool,
ABC_ENDPOINT,
successor,
foreground_runtime_id=99,
now=60100,
purpose=BACKGROUND_DUTY)
assert action == (
oldest_key,
oldest_attempt,
RECLAIMED_MAIN_RESUME)
assert pool.slots[newer_key.index].phase == LIVE
blocked = LivePool()
add_live_pool_attempt(
blocked,
live_ticket(6, 60),
106,
MAIN,
now=0)
blocked.learned_limit = 1
assert plan_main_replacement(
blocked,
ABC_ENDPOINT,
live_ticket(7, 70),
foreground_runtime_id=99,
now=60000,
purpose=BACKGROUND_DUTY,
admissible_transfer_waiting=True) is None
bootstrap = deepcopy(blocked)
action = plan_main_replacement(
bootstrap,
ABC_ENDPOINT,
live_ticket(8, 70),
foreground_runtime_id=99,
now=1,
purpose=DEMAND_BOOTSTRAP,
admissible_transfer_waiting=False)
assert action is not None
assert bootstrap.reclaim.stage == RECLAIM_REQUESTED
bootstrap_key, bootstrap_attempt, _ = action
assert authorize_live_slot_reclaim(
bootstrap,
bootstrap_key,
bootstrap_attempt,
foreground_runtime_id=99) == (True, True)
assert bootstrap.reclaim.stage == RECLAIM_AUTHORIZED
assert release_live_slot(
bootstrap,
bootstrap_key,
bootstrap_attempt).successor == live_ticket(8, 70)
foreground = deepcopy(blocked)
action = plan_main_replacement(
foreground,
ABC_ENDPOINT,
live_ticket(9, 99),
foreground_runtime_id=99,
now=1,
purpose=FOREGROUND_RECOVERY)
assert action is not None
assert foreground.reclaim.stage == RECLAIM_REQUESTED
victim_key, victim_attempt, _ = action
assert authorize_live_slot_reclaim(
foreground,
victim_key,
victim_attempt,
foreground_runtime_id=victim_attempt.runtime_id) == (True, False)
assert foreground.slots[victim_key.index].phase == LIVE
assert foreground.reclaim is None
transfer_only = LivePool()
add_live_pool_attempt(
transfer_only,
live_ticket(10, 80),
108,
MEDIA,
now=0)
assert plan_main_replacement(
transfer_only,
ABC_ENDPOINT,
live_ticket(11, 90),
foreground_runtime_id=99,
now=60000,
purpose=BACKGROUND_DUTY) is None
def test_released_continuation_is_exclusive_except_due_foreground_main():
source_ticket = live_ticket(1, 2)
source_attempt = live_attempt(source_ticket, 101)
successor = live_ticket(2, 2)
pool = LivePool()
pool.demand_transfer_continuation = DemandTransferContinuation(
source_key=LiveSlotKey(ABC_ENDPOINT, 0, 1),
source_attempt=source_attempt,
successor=successor,
runtime_id=successor.runtime_id,
proxy_generation=successor.proxy_generation,
use=MEDIA,
stage=CONTINUATION_RELEASED)
foreground_transfer = live_ticket(3, 1)
urgent_background_main = live_ticket(4, 3)
maintenance = live_ticket(5, 4)
masked = (
(foreground_transfer, MEDIA, True, True),
(urgent_background_main, MAIN, True, True),
(maintenance, "maintenance", True, True),
(successor, MEDIA, True, True),
)
assert select_with_released_continuation(pool, masked, 1) == successor
assert pool.demand_transfer_continuation is not None
future_foreground = live_ticket(6, 1)
with_future = ((future_foreground, MAIN, True, False),) + masked
assert select_with_released_continuation(pool, with_future, 1) == successor
assert pool.demand_transfer_continuation is not None
stale_foreground = live_ticket(7, 1)
with_stale = ((stale_foreground, MAIN, False, True),) + masked
assert select_with_released_continuation(pool, with_stale, 1) == successor
assert pool.demand_transfer_continuation is not None
due_foreground = live_ticket(
8,
1,
purpose=RECLAIMED_MAIN_RESUME)
with_due = ((due_foreground, MAIN, True, True),) + masked
assert select_with_released_continuation(pool, with_due, 1) == (
due_foreground)
assert pool.demand_transfer_continuation is None
assert not transfer_admission_matches(
pool,
ABC_ENDPOINT,
successor,
MEDIA,
RELEASED_CONTINUATION)
def test_reducer_no_appdata_relay_success_sibling_failure():
attempt1 = Attempt(proxy_generation=1, proxy_epoch=1, attempt_id=1)
attempt2 = Attempt(proxy_generation=1, proxy_epoch=1, attempt_id=2)
current = Status()
current = reduce_status(current, Fact(Status(
phase=FAILED,
reason=NO_APPDATA,
error=True,
attempt=attempt1)))
assert current.reason == NO_APPDATA
assert not current.error
current = reduce_status(current, Fact(
Status(phase=CONNECTED, attempt=attempt2),
success_scope=RELAY))
connected = current
assert connected.phase == CONNECTED
assert connected.success_until > 1000
sibling_failure = Fact(Status(
phase=FAILED,
reason=NO_SERVERHELLO,
error=True,
attempt=attempt1))
assert reduce_status(current, sibling_failure) == connected
same_attempt_stall = Fact(Status(
phase=FAILED,
reason=MTP_TIMEOUT_AFTER_DATA,
error=True,
attempt=attempt2))
stalled = reduce_status(current, same_attempt_stall)
assert stalled != connected
assert stalled.reason == MTP_TIMEOUT_AFTER_DATA
assert not stalled.error
def test_old_generation_and_probe_facts_are_shadowed():
current = Status(
phase=CONNECTED,
attempt=Attempt(proxy_generation=2, proxy_epoch=1, attempt_id=1),
success_until=16000)
old_generation_failure = Fact(Status(
phase=FAILED,
reason=HMAC_MISMATCH,
error=True,
attempt=Attempt(proxy_generation=1, proxy_epoch=99, attempt_id=99)))
probe_failure = Fact(Status(
phase=FAILED,
reason=NO_SERVERHELLO,
error=True,
attempt=Attempt(
proxy_generation=2,
proxy_epoch=1,
attempt_id=2,
probe=True)))
assert reduce_status(current, old_generation_failure) == current
assert reduce_status(current, probe_failure) == current
def test_older_progress_fact_cannot_repaint_connected_status():
current = Status(
phase=CONNECTED,
attempt=Attempt(proxy_generation=1, proxy_epoch=2, attempt_id=4),
success_until=0)
stale_progress = Fact(Status(
phase=CHECKING,
attempt=Attempt(proxy_generation=1, proxy_epoch=2, attempt_id=3)))
stale_epoch_progress = Fact(Status(
phase=CHECKING,
attempt=Attempt(proxy_generation=1, proxy_epoch=1, attempt_id=99)))
assert reduce_status(current, stale_progress) == current
assert reduce_status(current, stale_epoch_progress) == current
def test_selected_status_success_epoch_shadows_late_failures():
current = Status(
phase=CONNECTED,
attempt=Attempt(
proxy_generation=1,
proxy_epoch=2,
success_epoch=1,
attempt_id=4),
success_until=16000)
stale_success_epoch_failure = Fact(Status(
phase=FAILED,
reason=NO_SERVERHELLO,
error=True,
attempt=Attempt(
proxy_generation=1,
proxy_epoch=2,
success_epoch=0,
attempt_id=99)))
fresh_success_epoch_progress = Fact(Status(
phase=CHECKING,
attempt=Attempt(
proxy_generation=1,
proxy_epoch=2,
success_epoch=2,
attempt_id=1)))
assert reduce_status(current, stale_success_epoch_failure) == current
assert reduce_status(current, fresh_success_epoch_progress) != current
def make_relay_report(
endpoint=ABC_ENDPOINT,
runtime_id=2,
proxy_generation=36,
attempt_id=168,
proxy_epoch=1,
success_epoch=0,
started_at=100):
return RelayReport(
endpoint=endpoint,
runtime_id=runtime_id,
proxy_generation=proxy_generation,
attempt_id=attempt_id,
proxy_epoch=proxy_epoch,
success_epoch=success_epoch,
started_at=started_at)
def test_health_promotion_preserves_exact_admission_lineage():
report = make_relay_report()
unrelated = replace(report, attempt_id=169)
store = CanonicalEndpointStore()
seed_admission(store, report)
state = store.state(report.endpoint)
identity = relay_proof_identity(report)
assert set(state.attempt_starts) == {report.attempt_id}
assert has_endpoint_attempt(state, identity)
assert report_relay_success(store, unrelated, now=150) == MISSING_ADMISSION
assert report_relay_success(store, report, now=200) == INSERTED
assert not state.attempt_starts
assert set(state.relay_proofs) == {identity}
def test_canonical_endpoint_abc_lifecycle():
a = make_relay_report(runtime_id=2, attempt_id=168, started_at=100)
b = make_relay_report(runtime_id=5, attempt_id=169, started_at=110)
c = make_relay_report(runtime_id=3, attempt_id=170, started_at=120)
serialized = CanonicalEndpointStore()
seed_admission(serialized, a)
serialized_state = serialized.state(ABC_ENDPOINT)
pool = LivePool()
a_ticket = live_ticket(
1,
a.runtime_id,
proxy_generation=a.proxy_generation)
a_key, a_attempt = open_pool_attempt(pool, a_ticket, a.attempt_id)
assert reserve_live_slot(
pool,
ABC_ENDPOINT,
live_ticket(2, b.runtime_id, b.proxy_generation)) == (
None,
SLOT_WAIT)
assert report_failure(serialized, a, NO_SERVERHELLO, now=100) == (
TERMINAL_ATTEMPT)
assert serialized_state.opening_pressure_reason == NO_SERVERHELLO
assert serialized_state.opening_pressure_retry_until == 15100
assert mark_live_slot_capacity_terminal(
pool,
a_key,
a_attempt,
NO_SERVERHELLO,
True)
assert release_live_slot(
pool,
a_key,
a_attempt) == LiveReleaseResult(slot_released=True)
b_key, wait = reserve_live_slot(
pool,
ABC_ENDPOINT,
live_ticket(2, b.runtime_id, b.proxy_generation))
assert wait == NONE
assert b_key is not None
store = CanonicalEndpointStore()
for report in (a, b, c):
seed_admission(store, report)
state = store.state(ABC_ENDPOINT)
assert len(state.attempt_starts) == 3
assert report_relay_success(store, a, now=1000) == INSERTED
assert report_relay_success(store, b, now=1100) == INSERTED
assert report_relay_success(store, c, now=1200) == INSERTED
identities = {
relay_proof_identity(a),
relay_proof_identity(b),
relay_proof_identity(c),
}
assert set(state.relay_proofs) == identities
assert not state.attempt_starts
assert state.proxy_epoch == 4
assert state.success_epoch == 3
assert state.relay_proven
assert state.healthy
assert state.last_relay_success_at == 1200
protected = punishment_signature(state)
assert report_relay_stall(store, a, now=1300) == RETIRED_WITH_SURVIVORS
assert set(state.relay_proofs) == {
relay_proof_identity(b),
relay_proof_identity(c),
}
assert state.relay_proven
assert state.healthy
assert state.last_relay_success_at == 1200
assert punishment_signature(state) == protected
after_first_terminal = deepcopy(state)
assert report_relay_stall(store, a, now=1400) == MISSING_OR_DUPLICATE
assert state == after_first_terminal
def test_unowned_current_epoch_success_is_rejected():
store = CanonicalEndpointStore()
state = store.state(ABC_ENDPOINT)
state.generations[2] = 36
report = make_relay_report(
attempt_id=999,
proxy_epoch=state.proxy_epoch,
success_epoch=state.success_epoch,
started_at=500)
before = deepcopy(state)
assert not success_from_stale_attempt(report, state)
assert report_relay_success(store, report, now=1000) == MISSING_ADMISSION
assert state == before
def test_same_tuple_is_namespaced_by_canonical_endpoint():
first = make_relay_report(endpoint="first.example:443")
second = replace(first, endpoint="second.example:443")
store = CanonicalEndpointStore()
seed_admission(store, first)
seed_admission(store, second)
assert report_relay_success(store, first, now=1000) == INSERTED
assert store.state(first.endpoint).relay_proven
assert not store.state(second.endpoint).relay_proven
assert report_relay_success(store, second, now=1100) == INSERTED
assert set(store.state(first.endpoint).relay_proofs) == {
relay_proof_identity(first),
}
assert set(store.state(second.endpoint).relay_proofs) == {
relay_proof_identity(second),
}
def test_duplicate_success_has_no_growth_or_refresh():
report = make_relay_report()
store = CanonicalEndpointStore()
seed_admission(store, report)
assert report_relay_success(store, report, now=1000) == INSERTED
state = store.state(report.endpoint)
before = deepcopy(state)
assert report_relay_success(store, report, now=2000) == ALREADY_PROVEN
assert state == before
assert len(state.relay_proofs) == 1
assert state.relay_proofs[relay_proof_identity(report)] == RelayProofState(
proven_at=1000)
def test_sibling_preserving_ordinary_failure():
a = make_relay_report(runtime_id=2, attempt_id=168, started_at=100)
b = make_relay_report(runtime_id=5, attempt_id=169, started_at=110)
store = CanonicalEndpointStore()
for report in (a, b):
seed_admission(store, report)
assert report_relay_success(store, a, now=1000) == INSERTED
assert report_relay_success(store, b, now=1100) == INSERTED
state = store.state(ABC_ENDPOINT)
protected = punishment_signature(state)
assert report_failure(store, a, HMAC_MISMATCH, now=1200) == (
RETIRED_WITH_SURVIVORS)
assert set(state.relay_proofs) == {relay_proof_identity(b)}
assert state.relay_proven
assert state.healthy
assert state.last_relay_success_at == 1100
assert punishment_signature(state) == protected
def test_healthy_remote_close_retires_exact_proof_once():
a = make_relay_report(runtime_id=2, attempt_id=168, started_at=100)
b = make_relay_report(runtime_id=5, attempt_id=169, started_at=110)
store = CanonicalEndpointStore()
for report in (a, b):
seed_admission(store, report)
assert report_relay_success(store, a, now=1000) == INSERTED
assert report_relay_success(store, b, now=1100) == INSERTED
state = store.state(ABC_ENDPOINT)
protected = punishment_signature(state)
assert healthy_remote_close(store, a, now=1200) == (
RETIRED_WITH_SURVIVORS)
assert set(state.relay_proofs) == {relay_proof_identity(b)}
assert state.relay_proven
assert state.healthy
assert state.last_relay_success_at == 1100
assert punishment_signature(state) == protected
after_close = deepcopy(state)
assert neutral_retire(
store,
a,
now=1300,
origin="cancel") == MISSING_OR_DUPLICATE
assert state == after_close
def test_final_proof_real_failure_keeps_existing_punishment():
report = make_relay_report()
store = CanonicalEndpointStore()
seed_admission(store, report)
assert report_relay_success(store, report, now=1000) == INSERTED
state = store.state(report.endpoint)
route_failures = state.route_failure_count
cooldowns = state.cooldown_count
rotations = state.rotation_count
capability_failures = state.relay_capability_failure_count
assert report_failure(store, report, NO_MTPROTO, now=1200) == RETIRED_FINAL
assert not state.relay_proofs
assert not state.relay_proven
assert not state.last_relay_success_at
assert not state.healthy
assert state.route_state == "bad"
assert state.route_failure_count == route_failures + 1
assert state.cooldown_count == cooldowns + 1
assert state.cooldown_until == 16200
assert state.rotation_count == rotations + 1
assert state.relay_capability_failure_count == capability_failures + 1
assert not state.capability_relay_proven
def test_final_stall_only_invalidates_relay_capability():
report = make_relay_report()
store = CanonicalEndpointStore()
seed_admission(store, report)
assert report_relay_success(store, report, now=1000) == INSERTED
state = store.state(report.endpoint)
route_state = state.route_state
route_failures = state.route_failure_count
cooldowns = state.cooldown_count
rotations = state.rotation_count
capability_failures = state.relay_capability_failure_count
assert report_relay_stall(store, report, now=1200) == RETIRED_FINAL
assert not state.relay_proven
assert not state.last_relay_success_at
assert state.healthy
assert state.route_state == route_state
assert state.route_failure_count == route_failures
assert state.cooldown_count == cooldowns
assert state.rotation_count == rotations
assert state.relay_capability_failure_count == capability_failures + 1
assert not state.capability_relay_proven
def test_normal_close_cancel_and_owner_destruction_are_neutral():
for index, origin in enumerate((
"normal_close",
"cancel",
"owner_destruction")):
report = make_relay_report(
endpoint=f"neutral-{index}.example:443",
attempt_id=200 + index)
store = CanonicalEndpointStore()
seed_admission(store, report)
assert report_relay_success(store, report, now=1000) == INSERTED
state = store.state(report.endpoint)
protected = punishment_signature(state)
assert neutral_retire(
store,
report,
now=1200,
origin=origin) == RETIRED_FINAL
assert not state.relay_proven
assert not state.last_relay_success_at
assert state.healthy
assert state.capability_relay_proven
assert punishment_signature(state) == protected
def test_cancellation_after_handled_failure_is_noop():
report = make_relay_report()
store = CanonicalEndpointStore()
seed_admission(store, report)
assert report_relay_success(store, report, now=1000) == INSERTED
assert report_failure(store, report, NO_MTPROTO, now=1200) == RETIRED_FINAL
state = store.state(report.endpoint)
after_failure = deepcopy(state)
assert neutral_retire(
store,
report,
now=1300,
origin="cancel") == MISSING_OR_DUPLICATE
assert state == after_failure
def test_proxy_switch_generation_cleanup_is_runtime_scoped():
a = make_relay_report(runtime_id=2, attempt_id=168)
b = make_relay_report(runtime_id=5, attempt_id=169)
store = CanonicalEndpointStore()
for report in (a, b):
seed_admission(store, report)
assert report_relay_success(store, a, now=1000) == INSERTED
assert report_relay_success(store, b, now=1100) == INSERTED
state = store.state(ABC_ENDPOINT)
protected = punishment_signature(state)
apply_proxy_generation(store, runtime_id=2, proxy_generation=37, now=1200)
assert state.generations[2] == 37
assert set(state.relay_proofs) == {relay_proof_identity(b)}
assert state.last_relay_success_at == 1100
assert state.relay_proven
assert state.healthy
assert punishment_signature(state) == protected
after_switch = deepcopy(state)
assert report_failure(store, a, HMAC_MISMATCH, now=1300) == STALE_GENERATION
assert state == after_switch
def test_runtime_unregister_cleanup_is_runtime_scoped():
a = make_relay_report(runtime_id=2, attempt_id=168)
b = make_relay_report(runtime_id=5, attempt_id=169)
pending_a = make_relay_report(runtime_id=2, attempt_id=172, started_at=200)
pending_b = make_relay_report(runtime_id=5, attempt_id=173, started_at=210)
store = CanonicalEndpointStore()
for report in (a, b):
seed_admission(store, report)
assert report_relay_success(store, a, now=1000) == INSERTED
assert report_relay_success(store, b, now=1100) == INSERTED
seed_admission(store, pending_a)
seed_admission(store, pending_b)
state = store.state(ABC_ENDPOINT)
unregister_runtime(store, runtime_id=2)
assert 2 not in state.generations
assert state.generations[5] == 36
assert set(state.relay_proofs) == {relay_proof_identity(b)}
assert set(state.attempt_starts) == {pending_b.attempt_id}
assert len(state.attempt_starts) == 1
assert state.relay_proven
assert state.healthy
assert state.last_relay_success_at == 1100
def test_unresolved_attempts_are_removed_by_generation_and_runtime_cleanup():
a = make_relay_report(runtime_id=2, attempt_id=172, started_at=100)
b = make_relay_report(runtime_id=5, attempt_id=173, started_at=110)
expired = make_relay_report(runtime_id=7, attempt_id=174, started_at=0)
store = CanonicalEndpointStore()
for report in (a, b, expired):
seed_admission(store, report)
state = store.state(ABC_ENDPOINT)
assert set(state.attempt_starts) == {
a.attempt_id,
b.attempt_id,
expired.attempt_id,
}
apply_proxy_generation(store, runtime_id=2, proxy_generation=37, now=200)
assert set(state.attempt_starts) == {b.attempt_id, expired.attempt_id}
unregister_runtime(store, runtime_id=5)
assert set(state.attempt_starts) == {expired.attempt_id}
prune_expired_endpoint_state(state, ATTEMPT_HARD_TTL + 1)
assert not state.attempt_starts
def test_live_proof_survives_ten_minutes_and_state_touches():
a = make_relay_report(runtime_id=2, attempt_id=168, started_at=100)
b = make_relay_report(runtime_id=5, attempt_id=169, started_at=110)
touch = make_relay_report(
runtime_id=7,
attempt_id=170,
started_at=601200)
store = CanonicalEndpointStore()
for report in (a, b):
seed_admission(store, report)
assert report_relay_success(store, a, now=1000) == INSERTED
assert report_relay_success(store, b, now=1100) == INSERTED
state = store.state(ABC_ENDPOINT)
identities = {relay_proof_identity(a), relay_proof_identity(b)}
late = 601200
snapshot = snapshot_endpoint_state(store, ABC_ENDPOINT, late)
assert set(snapshot.relay_proofs) == identities
seed_admission(store, touch)
prune_expired_endpoint_state(state, late + 1)
assert set(state.relay_proofs) == identities
assert state.last_relay_success_at == 1100
assert report_failure(store, a, HMAC_MISMATCH, now=late + 2) == (
RETIRED_WITH_SURVIVORS)
assert set(state.relay_proofs) == {relay_proof_identity(b)}
assert report_relay_stall(store, b, now=late + 3) == RETIRED_FINAL
assert not state.relay_proofs
assert not state.relay_proven
assert state.healthy
def test_repeated_success_and_terminal_cycles_return_to_empty():
store = CanonicalEndpointStore()
count = 128
for index in range(count):
report = make_relay_report(
runtime_id=7,
attempt_id=1000 + index,
started_at=100 + index)
seed_admission(store, report)
assert report_relay_success(store, report, now=1000 + index) == INSERTED
assert report_relay_success(
store,
report,
now=2000 + index) == ALREADY_PROVEN
assert neutral_retire(
store,
report,
now=3000 + index,
origin="normal_close") == RETIRED_FINAL
assert neutral_retire(
store,
report,
now=4000 + index,
origin="cancel") == MISSING_OR_DUPLICATE
state = store.state(ABC_ENDPOINT)
assert not state.attempt_starts
assert not state.relay_proofs
assert not state.relay_proven
def test_retained_old_generation_membership_cannot_override_rejection():
report = make_relay_report()
store = CanonicalEndpointStore()
seed_admission(store, report)
assert report_relay_success(store, report, now=1000) == INSERTED
state = store.state(report.endpoint)
state.generations[report.runtime_id] = 37
assert relay_proof_identity(report) in state.relay_proofs
assert failure_from_stale_attempt(report, state)
assert success_from_stale_attempt(report, state)
before = deepcopy(state)
assert report_failure(store, report, HMAC_MISMATCH, now=1200) == (
STALE_GENERATION)
assert state == before
def test_generation_zero_is_stale_after_generation_36():
store = CanonicalEndpointStore()
state = store.state(ABC_ENDPOINT)
state.generations[2] = 36
report = make_relay_report(proxy_generation=0)
before = deepcopy(state)
assert report_relay_success(store, report, now=1000) == STALE_GENERATION
assert state == before
assert neutral_retire(store, report, now=1100) == STALE_GENERATION
assert state == before
def test_faketls_appdata_does_not_prove_relay_or_bump_epoch():
report = make_relay_report(proxy_generation=1)
store = CanonicalEndpointStore()
seed_admission(store, report)
state = store.state(report.endpoint)
identity = relay_proof_identity(report)
pool = LivePool()
ticket = live_ticket(
1,
report.runtime_id,
proxy_generation=report.proxy_generation)
key, attempt = open_pool_attempt(pool, ticket, report.attempt_id)
assert faketls_appdata_success(store, report, now=200)
assert not state.relay_proven
assert not state.relay_proofs
assert state.proxy_epoch == 1
assert state.success_epoch == 0
assert not state.last_relay_success_at
assert state.last_success_at == 200
assert set(state.attempt_starts) == {report.attempt_id}
assert has_endpoint_attempt(state, identity)
assert pool.slots[key.index].phase == OPENING
assert pool.opening_attempt == attempt
blocked = replace(report, attempt_id=169, started_at=210)
blocked_ticket = live_ticket(
2,
blocked.runtime_id,
proxy_generation=blocked.proxy_generation)
assert reserve_live_slot(pool, ABC_ENDPOINT, blocked_ticket) == (
None,
SLOT_WAIT)
assert set(state.attempt_starts) == {report.attempt_id}
assert report_relay_success(store, report, now=300) == INSERTED
assert mark_live_slot_relay_ready(pool, key, attempt, 300)
seed_admission(store, blocked)
blocked_key, wait = reserve_live_slot(
pool,
ABC_ENDPOINT,
blocked_ticket)
assert wait == NONE
assert blocked_key is not None
assert state.relay_proven
assert set(state.relay_proofs) == {identity}
assert set(state.attempt_starts) == {blocked.attempt_id}
assert pool.opening_ticket == blocked_ticket
assert state.proxy_epoch == 2
assert state.success_epoch == 1
assert state.last_relay_success_at == 300
def test_probe_waiting_slot_and_relay_proven_gate():
fresh_not_proven = {
"healthy": True,
"half_open": False,
"relay_proven": False,
"last_relay_success_at": 500,
}
fresh_proven = dict(fresh_not_proven, relay_proven=True)
assert probe_decision(True, fresh_proven) == "waiting_for_connection_slot"
assert probe_decision(False, fresh_not_proven) == "start_probe"
assert probe_decision(False, fresh_proven) == "connected_by_active_session"
def test_broker_claim_prevents_double_admit():
request = BrokerRequest()
assert broker_claim_front(request)
assert request.admission_in_progress
assert not broker_claim_front(request)
broker_complete_start(request)
assert request.start_scheduled
assert not broker_claim_front(request)
def test_broker_cancel_releases_admitted_lease_before_timer_fires():
request = BrokerRequest(admission=True, admission_lease_active=True)
broker_cancel(request)
assert not request.active
assert not request.admission
assert not request.admission_lease_active
assert not request.admission_in_progress
def test_capability_cache_soft_warning_vs_degraded_relay_failure():
assert capability_relay_proven_after_failure(
relay_proven=True,
reason=NO_APPDATA,
degraded=False)
assert not capability_relay_proven_after_failure(
relay_proven=True,
reason=NO_APPDATA,
degraded=True)
def test_capability_cache_relay_proof_ages_out():
assert capability_relay_proven_after_age(
relay_proven=True,
proven_at=100,
now=200,
ttl=1000)
assert not capability_relay_proven_after_age(
relay_proven=True,
proven_at=100,
now=2000,
ttl=1000)
assert not capability_relay_proven_after_age(
relay_proven=False,
proven_at=100,
now=200,
ttl=1000)
def run_all_truth_tables():
test_live_pool_bootstrap_and_sequential_expansion()
test_capacity_learning_accepts_only_exact_stable_frontier_evidence()
test_learned_cap_and_reclaim_are_serialized_until_exact_release()
test_demand_owned_continuation_survives_exact_main_proof_retirement()
test_reclaim_victim_order_and_foreground_main_protection()
test_cleanup_is_no_resume_closing_and_late_events_are_incarnation_safe()
test_reclaimed_resume_tracks_current_foreground_and_background_priority()
test_released_continuation_is_exclusive_except_due_foreground_main()
test_reducer_no_appdata_relay_success_sibling_failure()
test_old_generation_and_probe_facts_are_shadowed()
test_older_progress_fact_cannot_repaint_connected_status()
test_selected_status_success_epoch_shadows_late_failures()
test_health_promotion_preserves_exact_admission_lineage()
test_canonical_endpoint_abc_lifecycle()
test_unowned_current_epoch_success_is_rejected()
test_same_tuple_is_namespaced_by_canonical_endpoint()
test_duplicate_success_has_no_growth_or_refresh()
test_sibling_preserving_ordinary_failure()
test_healthy_remote_close_retires_exact_proof_once()
test_final_proof_real_failure_keeps_existing_punishment()
test_final_stall_only_invalidates_relay_capability()
test_normal_close_cancel_and_owner_destruction_are_neutral()
test_cancellation_after_handled_failure_is_noop()
test_proxy_switch_generation_cleanup_is_runtime_scoped()
test_runtime_unregister_cleanup_is_runtime_scoped()
test_unresolved_attempts_are_removed_by_generation_and_runtime_cleanup()
test_live_proof_survives_ten_minutes_and_state_touches()
test_repeated_success_and_terminal_cycles_return_to_empty()
test_retained_old_generation_membership_cannot_override_rejection()
test_generation_zero_is_stale_after_generation_36()
test_faketls_appdata_does_not_prove_relay_or_bump_epoch()
test_probe_waiting_slot_and_relay_proven_gate()
test_broker_claim_prevents_double_admit()
test_broker_cancel_releases_admitted_lease_before_timer_fires()
test_capability_cache_soft_warning_vs_degraded_relay_failure()
test_capability_cache_relay_proof_ages_out()