mirror of
https://github.com/Av0idD4rk/packmate.git
synced 2026-10-23 16:31:01 +00:00
428 lines
16 KiB
Python
428 lines
16 KiB
Python
import argparse
|
|
import base64
|
|
import binascii
|
|
import json
|
|
import math
|
|
import os
|
|
import struct
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
from network_load import PAYLOAD_MARKER, build_payload
|
|
|
|
|
|
BASE_URL = os.getenv("STRESS_API_URL", "http://172.31.250.10:65000/api").rstrip("/")
|
|
USER = os.getenv("STRESS_API_USER", "stress")
|
|
PASSWORD = os.getenv("STRESS_API_PASSWORD", "stress")
|
|
AUTHORIZATION = "Basic " + base64.b64encode(f"{USER}:{PASSWORD}".encode()).decode()
|
|
EXPECTED_LOCAL_IP = os.getenv("STRESS_TARGET_HOST", "172.31.250.10")
|
|
|
|
|
|
def request(method: str, path: str, body=None, timeout: float = 10):
|
|
payload = None if body is None else json.dumps(body).encode()
|
|
headers = {"Authorization": AUTHORIZATION}
|
|
if payload is not None:
|
|
headers["Content-Type"] = "application/json"
|
|
req = urllib.request.Request(BASE_URL + path, data=payload, headers=headers, method=method)
|
|
with urllib.request.urlopen(req, timeout=timeout) as response:
|
|
data = response.read()
|
|
return json.loads(data) if data else None
|
|
|
|
|
|
def wait_and_prepare(timeout: int) -> None:
|
|
deadline = time.monotonic() + timeout
|
|
while True:
|
|
try:
|
|
request("GET", "/service/")
|
|
break
|
|
except (urllib.error.URLError, TimeoutError):
|
|
if time.monotonic() >= deadline:
|
|
raise RuntimeError("Packmate API did not become ready")
|
|
time.sleep(1)
|
|
|
|
services = request("GET", "/service/")
|
|
if not any(service["port"] == 18080 for service in services):
|
|
request("POST", "/service/", {
|
|
"port": 18080,
|
|
"name": "network-stress",
|
|
"decryptTls": False,
|
|
"http": False,
|
|
"urldecodeHttpRequests": False,
|
|
"mergeAdjacentPackets": False,
|
|
"parseWebSockets": False,
|
|
})
|
|
patterns = request("GET", "/pattern/")
|
|
if not any(pattern["name"] == "network-stress-marker" for pattern in patterns):
|
|
request("POST", "/pattern/", {
|
|
"name": "network-stress-marker",
|
|
"value": PAYLOAD_MARKER.hex(),
|
|
"color": "#ff00ff",
|
|
"searchType": "SUBBYTES",
|
|
"searchTarget": "CONTENT",
|
|
"directionType": "BOTH",
|
|
"actionType": "FIND",
|
|
"serviceId": 18080,
|
|
})
|
|
if not request("GET", "/pcap/started"):
|
|
request("POST", "/pcap/start")
|
|
time.sleep(2)
|
|
|
|
|
|
def probe(
|
|
requests: int,
|
|
pause: float,
|
|
report: str,
|
|
max_p99_ms: float,
|
|
stop_file: str | None,
|
|
min_requests: int,
|
|
max_queue: int,
|
|
abort_file: str | None,
|
|
) -> None:
|
|
latencies = []
|
|
failures = []
|
|
max_capture_queue = 0
|
|
capture_drops = 0
|
|
interface_drops = 0
|
|
application_drops = 0
|
|
body = {"startingFrom": None, "pageSize": 20, "pattern": None, "favorites": False}
|
|
for index in range(requests):
|
|
if stop_file and index >= min_requests and Path(stop_file).exists():
|
|
break
|
|
started = time.perf_counter()
|
|
try:
|
|
request("POST", "/stream/all", body)
|
|
latencies.append((time.perf_counter() - started) * 1000)
|
|
metrics = request("GET", "/pcap/metrics")
|
|
if not metrics["captureOpen"] or not metrics["statsAvailable"]:
|
|
raise RuntimeError(f"pcap statistics unavailable: {metrics.get('statsError')}")
|
|
max_capture_queue = max(max_capture_queue, metrics["maximumQueuedPackets"])
|
|
capture_drops = max(capture_drops, metrics["droppedPackets"])
|
|
interface_drops = max(interface_drops, metrics["interfaceDroppedPackets"])
|
|
application_drops = max(application_drops, metrics["applicationDroppedPackets"])
|
|
if (
|
|
metrics["maximumQueuedPackets"] > max_queue
|
|
or metrics["droppedPackets"] > 0
|
|
or metrics["interfaceDroppedPackets"] > 0
|
|
or metrics["applicationDroppedPackets"] > 0
|
|
):
|
|
raise RuntimeError(
|
|
"capture limit exceeded: "
|
|
f"maxQueueObserved={metrics['maximumQueuedPackets']} maxQueue={max_queue} "
|
|
f"drops={metrics['droppedPackets']} interfaceDrops={metrics['interfaceDroppedPackets']} "
|
|
f"applicationDrops={metrics['applicationDroppedPackets']}"
|
|
)
|
|
except Exception as exc:
|
|
failures.append(f"request={index}: {type(exc).__name__}: {exc}")
|
|
if abort_file:
|
|
Path(abort_file).touch()
|
|
break
|
|
time.sleep(pause)
|
|
|
|
latencies.sort()
|
|
p99 = latencies[max(0, math.ceil(len(latencies) * 0.99) - 1)] if latencies else math.inf
|
|
result = {
|
|
"maximumRequests": requests,
|
|
"requests": len(latencies) + len(failures),
|
|
"successfulRequests": len(latencies),
|
|
"failedRequests": len(failures),
|
|
"latencyP99Millis": round(p99, 3) if latencies else None,
|
|
"latencyMaxMillis": round(max(latencies, default=0), 3),
|
|
"maxCaptureQueue": max_capture_queue,
|
|
"captureDroppedPackets": capture_drops,
|
|
"interfaceDroppedPackets": interface_drops,
|
|
"applicationDroppedPackets": application_drops,
|
|
"sampleErrors": failures[:20],
|
|
"stoppedByWorkloadCompletion": bool(stop_file and Path(stop_file).exists()),
|
|
}
|
|
Path(report).write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
|
|
print(json.dumps(result), flush=True)
|
|
if (
|
|
failures
|
|
or p99 > max_p99_ms
|
|
or capture_drops > 0
|
|
or interface_drops > 0
|
|
or application_drops > 0
|
|
or max_capture_queue > max_queue
|
|
or len(latencies) < min_requests
|
|
or (stop_file and not Path(stop_file).exists())
|
|
):
|
|
raise SystemExit(1)
|
|
|
|
|
|
def monitor_capture(
|
|
report: str,
|
|
stop_file: str,
|
|
abort_file: str,
|
|
max_queue: int,
|
|
pause: float,
|
|
min_samples: int,
|
|
) -> None:
|
|
samples = 0
|
|
maximum_queue = 0
|
|
maximum_drops = 0
|
|
maximum_interface_drops = 0
|
|
maximum_application_drops = 0
|
|
failures = []
|
|
last_metrics = None
|
|
started_at = int(time.time() * 1000)
|
|
|
|
while True:
|
|
try:
|
|
metrics = request("GET", "/pcap/metrics")
|
|
last_metrics = metrics
|
|
samples += 1
|
|
maximum_queue = max(maximum_queue, metrics["maximumQueuedPackets"])
|
|
maximum_drops = max(maximum_drops, metrics["droppedPackets"])
|
|
maximum_interface_drops = max(maximum_interface_drops, metrics["interfaceDroppedPackets"])
|
|
maximum_application_drops = max(
|
|
maximum_application_drops, metrics["applicationDroppedPackets"]
|
|
)
|
|
if not metrics["captureOpen"] or not metrics["statsAvailable"]:
|
|
raise RuntimeError(f"pcap statistics unavailable: {metrics.get('statsError')}")
|
|
if (
|
|
maximum_queue > max_queue
|
|
or maximum_drops > 0
|
|
or maximum_interface_drops > 0
|
|
or maximum_application_drops > 0
|
|
):
|
|
raise RuntimeError(
|
|
"capture limit exceeded: "
|
|
f"maxQueue={maximum_queue} limit={max_queue} "
|
|
f"drops={maximum_drops} interfaceDrops={maximum_interface_drops} "
|
|
f"applicationDrops={maximum_application_drops}"
|
|
)
|
|
except Exception as exc:
|
|
failures.append(f"{type(exc).__name__}: {exc}")
|
|
Path(abort_file).touch()
|
|
break
|
|
|
|
if samples >= min_samples and Path(stop_file).exists():
|
|
break
|
|
time.sleep(pause)
|
|
|
|
result = {
|
|
"startedAtUnixMillis": started_at,
|
|
"finishedAtUnixMillis": int(time.time() * 1000),
|
|
"samples": samples,
|
|
"maxCaptureQueue": maximum_queue,
|
|
"captureDroppedPackets": maximum_drops,
|
|
"interfaceDroppedPackets": maximum_interface_drops,
|
|
"applicationDroppedPackets": maximum_application_drops,
|
|
"queueLimit": max_queue,
|
|
"stoppedByWorkloadCompletion": Path(stop_file).exists(),
|
|
"abortedWorkload": Path(abort_file).exists(),
|
|
"lastMetrics": last_metrics,
|
|
"sampleErrors": failures[:20],
|
|
}
|
|
Path(report).write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
|
|
print(json.dumps(result), flush=True)
|
|
if (
|
|
failures
|
|
or samples < min_samples
|
|
or not Path(stop_file).exists()
|
|
or maximum_queue > max_queue
|
|
or maximum_drops > 0
|
|
or maximum_interface_drops > 0
|
|
or maximum_application_drops > 0
|
|
):
|
|
raise SystemExit(1)
|
|
|
|
|
|
def capture_metrics(report: str, require_empty_queue: bool, wait_seconds: int, max_queue: int | None) -> None:
|
|
deadline = time.monotonic() + wait_seconds
|
|
while True:
|
|
metrics = request("GET", "/pcap/metrics")
|
|
if not require_empty_queue or (metrics["queuedPackets"] == 0 and metrics["activeProcessors"] == 0):
|
|
break
|
|
if time.monotonic() >= deadline:
|
|
break
|
|
time.sleep(1)
|
|
Path(report).write_text(json.dumps(metrics, indent=2) + "\n", encoding="utf-8")
|
|
print(json.dumps(metrics), flush=True)
|
|
if (
|
|
not metrics["captureOpen"]
|
|
or not metrics["statsAvailable"]
|
|
or metrics["receivedPackets"] <= 0
|
|
or metrics["droppedPackets"] > 0
|
|
or metrics["interfaceDroppedPackets"] > 0
|
|
or metrics["applicationDroppedPackets"] > 0
|
|
or (max_queue is not None and metrics["maximumQueuedPackets"] > max_queue)
|
|
):
|
|
raise SystemExit(1)
|
|
if require_empty_queue and (metrics["queuedPackets"] != 0 or metrics["activeProcessors"] != 0):
|
|
raise SystemExit(1)
|
|
|
|
|
|
def parse_frames(content: bytes) -> list[bytes]:
|
|
frames = []
|
|
offset = 0
|
|
while offset < len(content):
|
|
if offset + 4 > len(content):
|
|
raise ValueError("truncated frame header")
|
|
size = struct.unpack_from("!I", content, offset)[0]
|
|
offset += 4
|
|
if size <= 0 or offset + size > len(content):
|
|
raise ValueError(f"invalid frame size {size}")
|
|
frames.append(content[offset:offset + size])
|
|
offset += size
|
|
return frames
|
|
|
|
|
|
def get_all_packets(stream_id: int) -> list[dict]:
|
|
packets = []
|
|
cursor = None
|
|
page_size = 1000
|
|
while True:
|
|
page = request("POST", f"/packet/{stream_id}", {
|
|
"startingFrom": cursor,
|
|
"pageSize": page_size,
|
|
})
|
|
packets.extend(page)
|
|
if len(page) < page_size:
|
|
return packets
|
|
new_cursor = page[-1]["id"]
|
|
if new_cursor == cursor:
|
|
raise ValueError("packet pagination cursor did not advance")
|
|
cursor = new_cursor
|
|
|
|
|
|
def verify_payloads(
|
|
run_id: int,
|
|
sample_size: int,
|
|
frames_per_stream: int,
|
|
payload_bytes: int,
|
|
report: str,
|
|
) -> None:
|
|
streams = request("POST", "/stream/all", {
|
|
"startingFrom": None,
|
|
"pageSize": sample_size,
|
|
"pattern": None,
|
|
"favorites": False,
|
|
})
|
|
failures = []
|
|
connection_ids = set()
|
|
verified = 0
|
|
for stream in streams:
|
|
try:
|
|
packets = get_all_packets(stream["id"])
|
|
incoming = b"".join(base64.b64decode(packet["content"]) for packet in packets if packet["incoming"])
|
|
outgoing = b"".join(base64.b64decode(packet["content"]) for packet in packets if not packet["incoming"])
|
|
incoming_frames = parse_frames(incoming)
|
|
outgoing_frames = parse_frames(outgoing)
|
|
if len(incoming_frames) != frames_per_stream or len(outgoing_frames) != frames_per_stream:
|
|
raise ValueError(
|
|
f"frame count incoming={len(incoming_frames)} outgoing={len(outgoing_frames)}"
|
|
)
|
|
|
|
identities = []
|
|
for sequence, payload in enumerate(incoming_frames):
|
|
if len(payload) < 24:
|
|
raise ValueError("short request payload")
|
|
frame_run, connection, shard, frame_sequence = struct.unpack("!QQII", payload[:24])
|
|
if frame_run != run_id or frame_sequence != sequence:
|
|
raise ValueError(
|
|
f"unexpected marker run={frame_run} connection={connection} sequence={frame_sequence}"
|
|
)
|
|
identities.append((connection, shard))
|
|
expected = build_payload(run_id, connection, shard, sequence, payload_bytes)
|
|
if payload != expected:
|
|
raise ValueError(f"request payload differs at sequence {sequence}")
|
|
if outgoing_frames[sequence] != expected:
|
|
raise ValueError(f"response payload differs at sequence {sequence}")
|
|
if len(set(identities)) != 1:
|
|
raise ValueError("one captured TCP stream contains multiple generator connections")
|
|
if identities[0] in connection_ids:
|
|
raise ValueError("duplicate generator connection across captured streams")
|
|
|
|
for packet in packets:
|
|
if not packet.get("sourceIp") or not packet.get("destinationIp"):
|
|
raise ValueError("missing packet IP metadata")
|
|
local_ip = packet["destinationIp"] if packet["incoming"] else packet["sourceIp"]
|
|
if local_ip != EXPECTED_LOCAL_IP:
|
|
raise ValueError(f"unexpected local IP {local_ip}")
|
|
|
|
connection_ids.add(identities[0])
|
|
verified += 1
|
|
except (ValueError, KeyError, struct.error, binascii.Error) as exc:
|
|
failures.append(f"stream={stream.get('id')}: {type(exc).__name__}: {exc}")
|
|
|
|
result = {
|
|
"runId": run_id,
|
|
"requestedSample": sample_size,
|
|
"returnedStreams": len(streams),
|
|
"verifiedStreams": verified,
|
|
"failedStreams": len(failures),
|
|
"sampleErrors": failures[:50],
|
|
}
|
|
Path(report).write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
|
|
print(json.dumps(result), flush=True)
|
|
if len(streams) < sample_size or failures:
|
|
raise SystemExit(1)
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
prepare = subparsers.add_parser("prepare")
|
|
prepare.add_argument("--timeout", type=int, default=300)
|
|
latency = subparsers.add_parser("probe")
|
|
latency.add_argument("--requests", type=int, default=300)
|
|
latency.add_argument("--pause", type=float, default=0.1)
|
|
latency.add_argument("--report", required=True)
|
|
latency.add_argument("--max-p99-ms", type=float, default=500)
|
|
latency.add_argument("--stop-file")
|
|
latency.add_argument("--min-requests", type=int, default=1)
|
|
latency.add_argument("--max-queue", type=int, required=True)
|
|
latency.add_argument("--abort-file")
|
|
monitor = subparsers.add_parser("monitor")
|
|
monitor.add_argument("--report", required=True)
|
|
monitor.add_argument("--stop-file", required=True)
|
|
monitor.add_argument("--abort-file", required=True)
|
|
monitor.add_argument("--max-queue", type=int, required=True)
|
|
monitor.add_argument("--pause", type=float, default=0.25)
|
|
monitor.add_argument("--min-samples", type=int, default=2)
|
|
metrics = subparsers.add_parser("metrics")
|
|
metrics.add_argument("--report", required=True)
|
|
metrics.add_argument("--require-empty-queue", action="store_true")
|
|
metrics.add_argument("--wait-seconds", type=int, default=0)
|
|
metrics.add_argument("--max-queue", type=int)
|
|
verify = subparsers.add_parser("verify")
|
|
verify.add_argument("--run-id", type=int, required=True)
|
|
verify.add_argument("--sample-size", type=int, default=200)
|
|
verify.add_argument("--frames", type=int, required=True)
|
|
verify.add_argument("--payload-bytes", type=int, required=True)
|
|
verify.add_argument("--report", required=True)
|
|
args = parser.parse_args()
|
|
if args.command == "prepare":
|
|
wait_and_prepare(args.timeout)
|
|
elif args.command == "probe":
|
|
probe(
|
|
args.requests,
|
|
args.pause,
|
|
args.report,
|
|
args.max_p99_ms,
|
|
args.stop_file,
|
|
args.min_requests,
|
|
args.max_queue,
|
|
args.abort_file,
|
|
)
|
|
elif args.command == "monitor":
|
|
monitor_capture(
|
|
args.report,
|
|
args.stop_file,
|
|
args.abort_file,
|
|
args.max_queue,
|
|
args.pause,
|
|
args.min_samples,
|
|
)
|
|
elif args.command == "metrics":
|
|
capture_metrics(args.report, args.require_empty_queue, args.wait_seconds, args.max_queue)
|
|
else:
|
|
verify_payloads(args.run_id, args.sample_size, args.frames, args.payload_bytes, args.report)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|