Prepare Packmate for production CTF deployments
@ -1,6 +1,11 @@
|
||||
build
|
||||
out
|
||||
screenshots
|
||||
stress
|
||||
frontend/node_modules
|
||||
frontend/dist
|
||||
**/__pycache__
|
||||
**/*.pyc
|
||||
.*
|
||||
docker-compose.yml
|
||||
Dockerfile_*
|
||||
|
||||
40
.env.example
Normal file
@ -0,0 +1,40 @@
|
||||
# Capture mode: LIVE, FILE, or VIEW.
|
||||
PACKMATE_MODE=LIVE
|
||||
|
||||
# LIVE mode: set these to the interface carrying game traffic and its local IP.
|
||||
PACKMATE_INTERFACE=game
|
||||
PACKMATE_LOCAL_IP=10.20.1.1
|
||||
|
||||
# FILE mode: put the file in ./pcaps and set its name here.
|
||||
PACKMATE_PCAP_FILE=
|
||||
|
||||
# Web interface credentials.
|
||||
PACKMATE_WEB_LOGIN=BinaryBears
|
||||
PACKMATE_WEB_PASSWORD=change-me
|
||||
|
||||
# PostgreSQL credentials. Use a strong value on a real installation.
|
||||
PACKMATE_DB_PASSWORD=change-me-too
|
||||
|
||||
# Capture pipeline. One worker is the safe compatibility default.
|
||||
PACKMATE_CAPTURE_WORKERS=1
|
||||
PACKMATE_CAPTURE_QUEUE_CAPACITY=250000
|
||||
PACKMATE_CAPTURE_BUFFER_SIZE_BYTES=67108864
|
||||
PACKMATE_CAPTURE_SHUTDOWN_TIMEOUT_SECONDS=30
|
||||
PACKMATE_CONTAINER_STOP_GRACE_PERIOD=45s
|
||||
|
||||
# Retention cleanup for LIVE mode.
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_ENABLED=true
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL_SECONDS=10
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_MAX_RUN_SECONDS=5
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_COMMIT_GRACE_SECONDS=120
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_THRESHOLD=15
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_BATCH_SIZE=500
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_MAX_BATCHES_PER_RUN=20
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_MAX_PACKETS_PER_BATCH=20000
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_MAX_MATCHES_PER_BATCH=20000
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_MAX_BYTES_PER_BATCH=67108864
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_STATEMENT_TIMEOUT_SECONDS=5
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_LOCK_TIMEOUT_SECONDS=1
|
||||
|
||||
# Local image tag produced by Docker Compose.
|
||||
BUILD_TAG=latest
|
||||
3
.gitattributes
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
*.sh text eol=lf
|
||||
*.key text eol=lf
|
||||
gradlew text eol=lf
|
||||
@ -1,3 +1,85 @@
|
||||
stages:
|
||||
- test
|
||||
- build
|
||||
|
||||
backend-tests:
|
||||
image: eclipse-temurin:17-jdk
|
||||
stage: test
|
||||
variables:
|
||||
DOCKER_HOST: tcp://docker:2375
|
||||
DOCKER_TLS_CERTDIR: ""
|
||||
GIT_SUBMODULE_STRATEGY: normal
|
||||
DOCKER_API_VERSION: "1.44"
|
||||
services:
|
||||
- name: docker:29-dind
|
||||
alias: docker
|
||||
before_script:
|
||||
- chmod +x gradlew
|
||||
script:
|
||||
- ./gradlew --no-daemon test stressTest bootJar
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
||||
- if: '$CI_COMMIT_BRANCH'
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
|
||||
network-stress:
|
||||
image: docker:29-cli
|
||||
stage: test
|
||||
timeout: 1h
|
||||
variables:
|
||||
DOCKER_HOST: tcp://docker:2375
|
||||
DOCKER_TLS_CERTDIR: ""
|
||||
GIT_SUBMODULE_STRATEGY: normal
|
||||
services:
|
||||
- name: docker:29-dind
|
||||
alias: docker
|
||||
script:
|
||||
- sh stress/network/run.sh
|
||||
artifacts:
|
||||
when: always
|
||||
expire_in: 14 days
|
||||
paths:
|
||||
- build/reports/network-stress/
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
||||
when: manual
|
||||
allow_failure: true
|
||||
- if: '$CI_COMMIT_BRANCH || $CI_COMMIT_TAG'
|
||||
when: manual
|
||||
allow_failure: true
|
||||
|
||||
network-smoke:
|
||||
image: docker:29-cli
|
||||
stage: test
|
||||
timeout: 30m
|
||||
variables:
|
||||
DOCKER_HOST: tcp://docker:2375
|
||||
DOCKER_TLS_CERTDIR: ""
|
||||
GIT_SUBMODULE_STRATEGY: normal
|
||||
PACKMATE_NETWORK_STRESS_STREAMS: "1000"
|
||||
PACKMATE_NETWORK_STRESS_LIVE_STREAMS: "200"
|
||||
PACKMATE_NETWORK_STRESS_SHARDS: "4"
|
||||
PACKMATE_NETWORK_STRESS_CONCURRENCY: "32"
|
||||
PACKMATE_NETWORK_STRESS_FAVORITE_STREAMS: "10"
|
||||
PACKMATE_NETWORK_STRESS_MIN_STREAMS_PER_SECOND: "5"
|
||||
PACKMATE_NETWORK_STRESS_MIN_CLEANUP_STREAMS_PER_SECOND: "20"
|
||||
PACKMATE_NETWORK_STRESS_MAX_CAPTURE_QUEUE: "50000"
|
||||
PACKMATE_NETWORK_STRESS_JAVA_MAX_HEAP: "2g"
|
||||
services:
|
||||
- name: docker:29-dind
|
||||
alias: docker
|
||||
script:
|
||||
- sh stress/network/run.sh
|
||||
artifacts:
|
||||
when: always
|
||||
expire_in: 14 days
|
||||
paths:
|
||||
- build/reports/network-stress/
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
||||
- if: '$CI_COMMIT_BRANCH'
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
|
||||
docker-build:
|
||||
image: docker:latest
|
||||
stage: build
|
||||
@ -21,4 +103,4 @@ docker-build:
|
||||
- docker compose push
|
||||
only:
|
||||
- master
|
||||
- tags
|
||||
- tags
|
||||
|
||||
4
.gitmodules
vendored
@ -1,4 +0,0 @@
|
||||
[submodule "frontend"]
|
||||
path = frontend
|
||||
url = git@github.com:Av0idD4rk/packmate.git
|
||||
branch = frontend
|
||||
36
README.md
@ -42,35 +42,25 @@ IP-адреса сохраняются вместе с пакетами, воз
|
||||
неограниченной распаковки GZIP HTTP находится в
|
||||
[poc/http_gzip_expansion.py](poc/http_gzip_expansion.py).
|
||||
|
||||
Для `LIVE` режима добавлена возобновляемая очистка старого трафика с настраиваемым retention,
|
||||
ограниченными транзакциями и отдельным stress test. Настройки и результаты проверки описаны в
|
||||
[docs/CLEANUP.md](docs/CLEANUP.md).
|
||||
|
||||
## Быстрый запуск
|
||||
Для быстрого запуска Packmate следует использовать [этот стартер](https://gitlab.com/packmate/starter/-/blob/master/README.md).
|
||||
|
||||
## Полный запуск
|
||||
Ниже следует инструкция для тех, кто хочет собрать Packmate самостоятельно.
|
||||
Нужны Docker Engine и плагин Docker Compose. Репозиторий уже содержит исходники backend и frontend,
|
||||
поэтому достаточно обычного клонирования без git submodules:
|
||||
|
||||
### Клонирование
|
||||
Поскольку этот репозиторий содержит фронтенд как git submodule, его необходимо клонировать так:
|
||||
```bash
|
||||
git clone --recurse-submodules git@github.com:Av0idD4rk/packmate.git
|
||||
|
||||
# Или, на старых версиях git
|
||||
git clone --recursive git@github.com:Av0idD4rk/packmate.git
|
||||
git clone https://github.com/Av0idD4rk/packmate.git
|
||||
cd packmate
|
||||
cp .env.example .env
|
||||
${EDITOR:-nano} .env
|
||||
docker compose up --build -d
|
||||
```
|
||||
|
||||
Если репозиторий уже был склонирован без подмодулей, необходимо выполнить:
|
||||
```bash
|
||||
git pull # Забираем свежую версию master из GitHub
|
||||
git submodule update --init --recursive
|
||||
```
|
||||
|
||||
### Настройка
|
||||
[Инструкция](docs/SETUP.md)
|
||||
|
||||
### Запуск
|
||||
После указания нужных настроек в env-файле, можно запустить приложение:
|
||||
```bash
|
||||
sudo docker compose up --build -d
|
||||
```
|
||||
В `.env` обязательно укажите `PACKMATE_INTERFACE` и `PACKMATE_LOCAL_IP` для режима `LIVE`, а также
|
||||
смените пароли. Описание всех режимов и настроек находится в [docs/SETUP.md](docs/SETUP.md).
|
||||
|
||||
При успешном запуске Packmate будет видно с любого хоста на порту `65000`.
|
||||
БД будет слушать на порту 65001, но будет разрешать подключения только с localhost.
|
||||
|
||||
35
README_EN.md
@ -23,38 +23,27 @@ Advanced network traffic flow analyzer for A/D CTFs.
|
||||
* Can automatically decompress GZIPed HTTP
|
||||
* Can automatically deflate WebSockets with permessages-deflate extension
|
||||
* Can automatically decrypt TLS with RSA using given private key (like Wireshark)
|
||||
* Bounded and resumable retention cleanup for long running `LIVE` captures
|
||||
|
||||

|
||||
|
||||
Cleanup settings and the PostgreSQL stress test are documented in [docs/CLEANUP.md](docs/CLEANUP.md).
|
||||
|
||||
## Quick Start
|
||||
To quickly start using Packmate, use [this starter](https://gitlab.com/packmate/starter/-/blob/master/README_EN.md).
|
||||
|
||||
## Full Build
|
||||
Below are the instructions for those who want to build Packmate on their own.
|
||||
Docker Engine and the Docker Compose plugin are required. The repository contains both backend and
|
||||
frontend sources, so a regular clone is enough and no git submodules are needed:
|
||||
|
||||
### Cloning
|
||||
As this repository contains frontend part as a git submodule, it has to be cloned like this:
|
||||
```bash
|
||||
git clone --recurse-submodules https://gitlab.com/packmate/Packmate.git
|
||||
|
||||
# Or if you have older git
|
||||
git clone --recursive https://gitlab.com/packmate/Packmate.git
|
||||
git clone https://github.com/Av0idD4rk/packmate.git
|
||||
cd packmate
|
||||
cp .env.example .env
|
||||
${EDITOR:-nano} .env
|
||||
docker compose up --build -d
|
||||
```
|
||||
|
||||
If the repository was already cloned without submodule, just run:
|
||||
```bash
|
||||
git pull
|
||||
git submodule update --init --recursive
|
||||
```
|
||||
|
||||
### Setup
|
||||
[Instructions](docs/SETUP_EN.md)
|
||||
|
||||
### Launch
|
||||
After filling in env file you can launch the app:
|
||||
```bash
|
||||
sudo docker-compose up --build -d
|
||||
```
|
||||
For `LIVE` mode, set `PACKMATE_INTERFACE` and `PACKMATE_LOCAL_IP` in `.env`, and change the
|
||||
passwords. See [docs/SETUP_EN.md](docs/SETUP_EN.md) for all modes and settings.
|
||||
|
||||
If everything went fine, Packmate will be available on port `65000` from any host.
|
||||
Database with listen on port 65001, but will only accept connections from localhost.
|
||||
|
||||
@ -53,9 +53,34 @@ dependencies {
|
||||
runtimeOnly("org.springframework.boot:spring-boot-devtools")
|
||||
runtimeOnly("org.postgresql:postgresql")
|
||||
annotationProcessor("org.projectlombok:lombok")
|
||||
testImplementation("org.junit.jupiter:junit-jupiter:5.9.2")
|
||||
testImplementation("org.springframework.boot:spring-boot-starter-test")
|
||||
testImplementation("org.testcontainers:junit-jupiter:1.21.4")
|
||||
testImplementation("org.testcontainers:postgresql:1.21.4")
|
||||
}
|
||||
|
||||
tasks.withType<Test>().configureEach {
|
||||
useJUnitPlatform()
|
||||
if (System.getenv("DOCKER_API_VERSION") == null) {
|
||||
environment("DOCKER_API_VERSION", "1.44")
|
||||
}
|
||||
}
|
||||
|
||||
tasks.getByName<Test>("test") {
|
||||
useJUnitPlatform()
|
||||
}
|
||||
useJUnitPlatform {
|
||||
excludeTags("stress")
|
||||
}
|
||||
}
|
||||
|
||||
tasks.register<Test>("stressTest") {
|
||||
group = "verification"
|
||||
description = "Runs the PostgreSQL cleanup stress test"
|
||||
testClassesDirs = sourceSets["test"].output.classesDirs
|
||||
classpath = sourceSets["test"].runtimeClasspath
|
||||
useJUnitPlatform {
|
||||
includeTags("stress")
|
||||
}
|
||||
testLogging {
|
||||
events("passed", "failed", "skipped")
|
||||
showStandardStreams = true
|
||||
}
|
||||
}
|
||||
|
||||
@ -8,12 +8,26 @@ services:
|
||||
PCAP_FILE: ${PACKMATE_PCAP_FILE:-}
|
||||
WEB_LOGIN: ${PACKMATE_WEB_LOGIN:-BinaryBears}
|
||||
WEB_PASSWORD: ${PACKMATE_WEB_PASSWORD:-123456}
|
||||
OLD_STREAMS_CLEANUP_ENABLED: ${PACKMATE_OLD_STREAMS_CLEANUP_ENABLED:-false}
|
||||
OLD_STREAMS_CLEANUP_INTERVAL: ${PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL:-5}
|
||||
OLD_STREAMS_CLEANUP_THRESHOLD: ${PACKMATE_OLD_STREAMS_CLEANUP_THRESHOLD:-240}
|
||||
CAPTURE_WORKERS: ${PACKMATE_CAPTURE_WORKERS:-1}
|
||||
CAPTURE_QUEUE_CAPACITY: ${PACKMATE_CAPTURE_QUEUE_CAPACITY:-250000}
|
||||
CAPTURE_BUFFER_SIZE_BYTES: ${PACKMATE_CAPTURE_BUFFER_SIZE_BYTES:-67108864}
|
||||
CAPTURE_SHUTDOWN_TIMEOUT_SECONDS: ${PACKMATE_CAPTURE_SHUTDOWN_TIMEOUT_SECONDS:-30}
|
||||
OLD_STREAMS_CLEANUP_ENABLED: ${PACKMATE_OLD_STREAMS_CLEANUP_ENABLED:-true}
|
||||
OLD_STREAMS_CLEANUP_INTERVAL_SECONDS: ${PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL_SECONDS:-10}
|
||||
OLD_STREAMS_CLEANUP_MAX_RUN_SECONDS: ${PACKMATE_OLD_STREAMS_CLEANUP_MAX_RUN_SECONDS:-5}
|
||||
OLD_STREAMS_CLEANUP_COMMIT_GRACE_SECONDS: ${PACKMATE_OLD_STREAMS_CLEANUP_COMMIT_GRACE_SECONDS:-120}
|
||||
OLD_STREAMS_CLEANUP_THRESHOLD: ${PACKMATE_OLD_STREAMS_CLEANUP_THRESHOLD:-15}
|
||||
OLD_STREAMS_CLEANUP_BATCH_SIZE: ${PACKMATE_OLD_STREAMS_CLEANUP_BATCH_SIZE:-500}
|
||||
OLD_STREAMS_CLEANUP_MAX_BATCHES_PER_RUN: ${PACKMATE_OLD_STREAMS_CLEANUP_MAX_BATCHES_PER_RUN:-20}
|
||||
OLD_STREAMS_CLEANUP_MAX_PACKETS_PER_BATCH: ${PACKMATE_OLD_STREAMS_CLEANUP_MAX_PACKETS_PER_BATCH:-20000}
|
||||
OLD_STREAMS_CLEANUP_MAX_MATCHES_PER_BATCH: ${PACKMATE_OLD_STREAMS_CLEANUP_MAX_MATCHES_PER_BATCH:-20000}
|
||||
OLD_STREAMS_CLEANUP_MAX_BYTES_PER_BATCH: ${PACKMATE_OLD_STREAMS_CLEANUP_MAX_BYTES_PER_BATCH:-67108864}
|
||||
OLD_STREAMS_CLEANUP_STATEMENT_TIMEOUT_SECONDS: ${PACKMATE_OLD_STREAMS_CLEANUP_STATEMENT_TIMEOUT_SECONDS:-5}
|
||||
OLD_STREAMS_CLEANUP_LOCK_TIMEOUT_SECONDS: ${PACKMATE_OLD_STREAMS_CLEANUP_LOCK_TIMEOUT_SECONDS:-1}
|
||||
env_file:
|
||||
- .env
|
||||
container_name: packmate-app
|
||||
stop_grace_period: ${PACKMATE_CONTAINER_STOP_GRACE_PERIOD:-45s}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/Dockerfile_app
|
||||
@ -27,6 +41,20 @@ services:
|
||||
condition: service_healthy
|
||||
db: # port = 65001
|
||||
container_name: packmate-db
|
||||
command:
|
||||
- postgres
|
||||
- -c
|
||||
- autovacuum_max_workers=4
|
||||
- -c
|
||||
- autovacuum_naptime=10s
|
||||
- -c
|
||||
- autovacuum_vacuum_threshold=1000
|
||||
- -c
|
||||
- autovacuum_vacuum_scale_factor=0.02
|
||||
- -c
|
||||
- autovacuum_analyze_scale_factor=0.05
|
||||
- -c
|
||||
- autovacuum_vacuum_cost_limit=1000
|
||||
environment:
|
||||
POSTGRES_USER: packmate
|
||||
POSTGRES_PASSWORD: ${PACKMATE_DB_PASSWORD:-K604YnL3G1hp2RDkCZNjGpxbyNpNHTRb}
|
||||
@ -43,4 +71,4 @@ services:
|
||||
test: [ "CMD-SHELL", "pg_isready -U packmate -p 65001" ]
|
||||
interval: 2s
|
||||
timeout: 5s
|
||||
retries: 15
|
||||
retries: 15
|
||||
|
||||
@ -7,7 +7,7 @@ FROM eclipse-temurin:17-jdk
|
||||
WORKDIR /tmp/compile/
|
||||
COPY ./ .
|
||||
COPY --from=0 /tmp/build/dist/ ./src/main/resources/static/
|
||||
RUN ./gradlew --no-daemon bootJar
|
||||
RUN sed -i 's/\r$//' gradlew && chmod +x gradlew && ./gradlew --no-daemon bootJar
|
||||
|
||||
FROM eclipse-temurin:17-jre
|
||||
WORKDIR /app
|
||||
@ -20,10 +20,23 @@ CMD [ "java", "-Djava.net.preferIPv4Stack=true", "-Djava.net.preferIPv4Addresses
|
||||
"--packmate.capture-mode=${MODE}", "--packmate.pcap-file=${PCAP_FILE}", \
|
||||
"--packmate.interface-name=${INTERFACE}", "--packmate.local-ip=${LOCAL_IP}", \
|
||||
"--packmate.web.account-login=${WEB_LOGIN}", "--packmate.web.account-password=${WEB_PASSWORD}", \
|
||||
"--packmate.capture.workers=${CAPTURE_WORKERS}", \
|
||||
"--packmate.capture.queue-capacity=${CAPTURE_QUEUE_CAPACITY}", \
|
||||
"--packmate.capture.buffer-size-bytes=${CAPTURE_BUFFER_SIZE_BYTES}", \
|
||||
"--packmate.capture.shutdown-timeout-seconds=${CAPTURE_SHUTDOWN_TIMEOUT_SECONDS}", \
|
||||
"--packmate.cleanup.enabled=${OLD_STREAMS_CLEANUP_ENABLED}", \
|
||||
"--packmate.cleanup.interval=${OLD_STREAMS_CLEANUP_INTERVAL}", \
|
||||
"--packmate.cleanup.interval-seconds=${OLD_STREAMS_CLEANUP_INTERVAL_SECONDS}", \
|
||||
"--packmate.cleanup.max-run-seconds=${OLD_STREAMS_CLEANUP_MAX_RUN_SECONDS}", \
|
||||
"--packmate.cleanup.commit-grace-seconds=${OLD_STREAMS_CLEANUP_COMMIT_GRACE_SECONDS}", \
|
||||
"--packmate.cleanup.threshold=${OLD_STREAMS_CLEANUP_THRESHOLD}", \
|
||||
"--packmate.cleanup.batch-size=${OLD_STREAMS_CLEANUP_BATCH_SIZE}", \
|
||||
"--packmate.cleanup.max-batches-per-run=${OLD_STREAMS_CLEANUP_MAX_BATCHES_PER_RUN}", \
|
||||
"--packmate.cleanup.max-packets-per-batch=${OLD_STREAMS_CLEANUP_MAX_PACKETS_PER_BATCH}", \
|
||||
"--packmate.cleanup.max-matches-per-batch=${OLD_STREAMS_CLEANUP_MAX_MATCHES_PER_BATCH}", \
|
||||
"--packmate.cleanup.max-bytes-per-batch=${OLD_STREAMS_CLEANUP_MAX_BYTES_PER_BATCH}", \
|
||||
"--packmate.cleanup.statement-timeout-seconds=${OLD_STREAMS_CLEANUP_STATEMENT_TIMEOUT_SECONDS}", \
|
||||
"--packmate.cleanup.lock-timeout-seconds=${OLD_STREAMS_CLEANUP_LOCK_TIMEOUT_SECONDS}", \
|
||||
"--server.port=65000", "--server.address=0.0.0.0" \
|
||||
]
|
||||
|
||||
EXPOSE 65000
|
||||
EXPOSE 65000
|
||||
|
||||
@ -579,15 +579,15 @@ log_timezone = 'UTC'
|
||||
# their durations, > 0 logs only
|
||||
# actions running at least this number
|
||||
# of milliseconds.
|
||||
#autovacuum_max_workers = 3 # max number of autovacuum subprocesses
|
||||
autovacuum_max_workers = 4 # packet cleanup creates dead tuples continuously
|
||||
# (change requires restart)
|
||||
#autovacuum_naptime = 1min # time between autovacuum runs
|
||||
#autovacuum_vacuum_threshold = 50 # min number of row updates before
|
||||
autovacuum_naptime = 10s # time between autovacuum runs
|
||||
autovacuum_vacuum_threshold = 1000 # min number of row updates before
|
||||
# vacuum
|
||||
#autovacuum_analyze_threshold = 50 # min number of row updates before
|
||||
# analyze
|
||||
#autovacuum_vacuum_scale_factor = 0.2 # fraction of table size before vacuum
|
||||
#autovacuum_analyze_scale_factor = 0.1 # fraction of table size before analyze
|
||||
autovacuum_vacuum_scale_factor = 0.02 # fraction of table size before vacuum
|
||||
autovacuum_analyze_scale_factor = 0.05 # fraction of table size before analyze
|
||||
#autovacuum_freeze_max_age = 200000000 # maximum XID age before forced vacuum
|
||||
# (change requires restart)
|
||||
#autovacuum_multixact_freeze_max_age = 400000000 # maximum multixact age
|
||||
@ -596,7 +596,7 @@ log_timezone = 'UTC'
|
||||
#autovacuum_vacuum_cost_delay = 2ms # default vacuum cost delay for
|
||||
# autovacuum, in milliseconds;
|
||||
# -1 means use vacuum_cost_delay
|
||||
#autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for
|
||||
autovacuum_vacuum_cost_limit = 1000 # let vacuum keep up with packet deletion
|
||||
# autovacuum, -1 means use
|
||||
# vacuum_cost_limit
|
||||
|
||||
|
||||
@ -1,2 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cp -f /tmp/postgresql.conf /var/lib/postgresql/data/postgresql.conf
|
||||
|
||||
126
docs/CLEANUP.md
Normal file
@ -0,0 +1,126 @@
|
||||
# Очистка старых потоков
|
||||
|
||||
В режиме `LIVE` Packmate периодически удаляет старые потоки, кроме отмеченных как favorite. Удаление выполняется короткими независимыми транзакциями напрямую в PostgreSQL и не загружает граф `Stream -> Packet -> FoundPattern` в Hibernate.
|
||||
|
||||
По умолчанию очистка включена в Docker Compose и сохраняет последние 15 минут:
|
||||
|
||||
| Переменная | Значение | Назначение |
|
||||
|---|---:|---|
|
||||
| `PACKMATE_OLD_STREAMS_CLEANUP_ENABLED` | `true` | Включает задачу |
|
||||
| `PACKMATE_OLD_STREAMS_CLEANUP_THRESHOLD` | `15` | Retention для non-favorite потоков в минутах |
|
||||
| `PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL_SECONDS` | `10` | Пауза между запусками в секундах |
|
||||
| `PACKMATE_OLD_STREAMS_CLEANUP_MAX_RUN_SECONDS` | `5` | После этого cleaner не начинает новые пачки |
|
||||
| `PACKMATE_OLD_STREAMS_CLEANUP_COMMIT_GRACE_SECONDS` | `120` | Минимальное время после записи до очистки |
|
||||
| `PACKMATE_OLD_STREAMS_CLEANUP_BATCH_SIZE` | `500` | Максимум потоков в транзакции |
|
||||
| `PACKMATE_OLD_STREAMS_CLEANUP_MAX_BATCHES_PER_RUN` | `20` | Максимум транзакций за один запуск |
|
||||
| `PACKMATE_OLD_STREAMS_CLEANUP_MAX_PACKETS_PER_BATCH` | `20000` | Бюджет пакетов на транзакцию |
|
||||
| `PACKMATE_OLD_STREAMS_CLEANUP_MAX_MATCHES_PER_BATCH` | `20000` | Бюджет найденных совпадений на транзакцию |
|
||||
| `PACKMATE_OLD_STREAMS_CLEANUP_MAX_BYTES_PER_BATCH` | `67108864` | Бюджет содержимого на транзакцию |
|
||||
| `PACKMATE_OLD_STREAMS_CLEANUP_STATEMENT_TIMEOUT_SECONDS` | `5` | Общий deadline одной cleanup-транзакции |
|
||||
| `PACKMATE_OLD_STREAMS_CLEANUP_LOCK_TIMEOUT_SECONDS` | `1` | Лимит ожидания блокировки |
|
||||
|
||||
PostgreSQL запускается с более частым autovacuum. Параметры заданы также в `docker-compose.yml`, поэтому применяются к уже существующему volume после перезапуска контейнера БД.
|
||||
|
||||
Перед удалением поток помечается как находящийся в очистке и перестаёт попадать в выдачу и lookback. Пакеты удаляются ограниченными кусками. Если приложение или БД перезапустятся посередине, следующий запуск продолжит такой поток. Поэтому один большой TCP stream не блокирует всю очередь очистки.
|
||||
|
||||
Retention и частота полностью настраиваются через `.env`. Например, чтобы хранить час истории и запускать cleaner каждые 30 секунд:
|
||||
|
||||
```dotenv
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_THRESHOLD=60
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL_SECONDS=30
|
||||
```
|
||||
|
||||
В старых конфигурациях интервал задавался в минутах через `PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL`.
|
||||
Теперь используется `PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL_SECONDS`; старую переменную нужно заменить.
|
||||
`MAX_RUN_SECONDS` ограничивает момент запуска следующей пачки. Внутри пачки оставшееся время SQL
|
||||
пересчитывается перед каждым запросом, поэтому вся транзакция ограничена `STATEMENT_TIMEOUT_SECONDS`.
|
||||
|
||||
Для соревнований effective cleanup throughput должен быть выше скорости, с которой трафик пересекает retention cutoff. Если Packmate начинает деградировать через 25–30 минут, безопасный исходный retention — 15 минут; favorite потоки не удаляются.
|
||||
|
||||
## Стресс тест
|
||||
|
||||
Тест всегда создаёт одноразовый PostgreSQL через Testcontainers. Рабочая база Packmate не используется.
|
||||
|
||||
```bash
|
||||
./gradlew stressTest
|
||||
```
|
||||
|
||||
Стандартный профиль создаёт 30 000 обычных потоков, oversized поток с ложной метаинформацией и более 900 000 пакетов. Он использует production лимиты запусков и паузы, параллельно запускает два writer и проверяет:
|
||||
|
||||
- удаление всех старых non-favorite данных и сохранение favorite/new данных;
|
||||
- отсутствие потерянных новых записей и orphan rows;
|
||||
- возобновление oversized потока несколькими ограниченными транзакциями;
|
||||
- p99 задержки ingest не выше 500 мс;
|
||||
- длительность одной cleanup пачки не выше 5 секунд;
|
||||
- скорость очистки не ниже 5 000 пакетов/с;
|
||||
- запуск autovacuum и снижение dead tuples до 50 000.
|
||||
|
||||
Отчёт сохраняется в `build/reports/cleanup-stress.json`. Размер нагрузки и пороги настраиваются переменными `PACKMATE_STRESS_*`, перечисленными в `OldStreamsCleanerStressTest`.
|
||||
|
||||
## Сетевой end-to-end stress test
|
||||
|
||||
`OldStreamsCleanerStressTest` изолирует производительность PostgreSQL. Отдельный Linux/Docker стенд
|
||||
проверяет весь production тракт: реальные TCP соединения через Docker `veth`, захват `libpcap`,
|
||||
сборку потоков, очередь обработчика, Hibernate, PostgreSQL, cleaner и REST API.
|
||||
|
||||
```bash
|
||||
sh stress/network/run.sh
|
||||
```
|
||||
|
||||
По умолчанию тест создаёт **250 000** исходных TCP соединений и ещё **50 000** соединений во время
|
||||
очистки. В каждом соединении передаётся восемь пар request/response с payload 1024 байта. Нагрузка
|
||||
распределяется между 16 контейнерами с отдельными source IP, чтобы один диапазон ephemeral ports
|
||||
не ограничивал результат. Это около 4,8 млн прикладных сообщений плюс TCP handshake, ACK и FIN.
|
||||
|
||||
Перед очисткой тест полностью выгружает ledger сохранённых потоков и для каждого независимо проверяет
|
||||
`run_id`, connection ID, shard, полную длину и MD5 payload в обоих направлениях, а также IP адреса.
|
||||
Дополнительно 200 потоков проверяются через REST API. После этого тест
|
||||
искусственно сдвигает timestamp уже захваченных строк за retention cutoff, чтобы проверить production
|
||||
cleaner без ожидания полного retention period, и одновременно продолжает подавать свежий сетевой трафик.
|
||||
Тестовый retention по умолчанию равен 1440 минутам, поэтому случайно свежие строки не попадают в cleanup.
|
||||
Live фаза длится не менее 20 секунд и PASS возможен только при измеренном пересечении commit новых
|
||||
потоков с удалением baseline.
|
||||
|
||||
Тест завершается ошибкой при любой ошибке генератора, потере сохранённого потока, повреждении
|
||||
проверяемого payload, `pcap` drop, capture queue более 500 000 пакетов, orphan row, незавершённом cleanup
|
||||
или REST p99 выше 500 мс. После baseline проверяется ранний gate, поэтому заведомо повреждённый прогон
|
||||
не маскирует потери длительным ожиданием ingest. Размер capture queue измеряется монотонным high water
|
||||
mark внутри приложения. При превышении лимита monitor создаёт общий abort marker, и все генераторы
|
||||
перестают открывать новые соединения. Cleaner должен удалять не менее 500 streams/s; порог настраивается.
|
||||
PostgreSQL создаётся в отдельном Compose project на
|
||||
одноразовом Docker volume; до нагрузки проверяется доступное место. Рабочая база Packmate не используется.
|
||||
JSON отчёты, effective config, overlap timeline и логи сохраняются в отдельном каталоге запуска под
|
||||
`build/reports/network-stress/`.
|
||||
|
||||
Контрольный запуск 23 октября 2026 года показал предел старого однопоточного capture path: при 250 000
|
||||
TCP соединений `libpcap` сообщил 1 967 297 drops, а неограниченная очередь выросла до 4 894 878 задач.
|
||||
После перехода на 12 flow-sharded workers повторный baseline сохранил 250 000 из 250 000 соединений и
|
||||
8 013 904 захваченных пакета без kernel, interface или application drops. Полный ledger подтвердил
|
||||
payload и метаданные всех 250 000 потоков; очередь полностью опустела, её high water составил 61 396
|
||||
из 500 004 задач. Следующая волна передала ещё 50 000 соединений во время cleanup без ошибок генератора
|
||||
и packet drops; её high water составил 120 570.
|
||||
|
||||
Отдельный завершённый профиль 5 000 + 1 000 прошёл весь ingest/cleanup/ledger цикл, включая измеренное
|
||||
пересечение cleaner с live traffic, сохранение favorite, REST p99 189,686 мс и отсутствие orphan rows.
|
||||
Production по умолчанию сохраняет `PACKMATE_CAPTURE_WORKERS=1` как безопасный режим совместимости;
|
||||
число workers нужно выбирать по результатам этого стенда на реальном сервере. Throughput генераторов
|
||||
измеряется от старта первого до завершения последнего workload-процесса и не включает создание Docker
|
||||
контейнеров.
|
||||
|
||||
Для длительного прогона на миллион соединений:
|
||||
|
||||
```bash
|
||||
PACKMATE_NETWORK_STRESS_STREAMS=1000000 \
|
||||
PACKMATE_NETWORK_STRESS_LIVE_STREAMS=100000 \
|
||||
PACKMATE_NETWORK_STRESS_SHARDS=16 \
|
||||
sh stress/network/run.sh
|
||||
```
|
||||
|
||||
Основные параметры: `PACKMATE_NETWORK_STRESS_STREAMS`, `PACKMATE_NETWORK_STRESS_LIVE_STREAMS`,
|
||||
`PACKMATE_NETWORK_STRESS_SHARDS`, `PACKMATE_NETWORK_STRESS_CONCURRENCY`,
|
||||
`PACKMATE_NETWORK_STRESS_FRAMES_PER_STREAM`, `PACKMATE_NETWORK_STRESS_PAYLOAD_BYTES`,
|
||||
`PACKMATE_NETWORK_STRESS_MAX_CAPTURE_QUEUE`, `PACKMATE_NETWORK_STRESS_MIN_LIVE_DURATION_SECONDS`,
|
||||
`PACKMATE_NETWORK_STRESS_MIN_CLEANUP_STREAMS_PER_SECOND` и таймауты
|
||||
`PACKMATE_NETWORK_STRESS_*_TIMEOUT_SECONDS`. `PACKMATE_NETWORK_STRESS_RUN_ID` задаёт имя каталога отчёта.
|
||||
В GitLab CI малый профиль `network-smoke` обязателен, а дорогой capacity профиль доступен как ручной job
|
||||
`network-stress`; отчёты обоих сохраняются как artifacts.
|
||||
@ -1,6 +1,12 @@
|
||||
## Настройка
|
||||
|
||||
Packmate использует настройки из файла `.env` (в той же папке, что и `docker-compose.yml`)
|
||||
Packmate использует настройки из файла `.env` в той же папке, что и `docker-compose.yml`.
|
||||
Создайте его из полного шаблона и измените значения под сервер:
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
${EDITOR:-nano} .env
|
||||
```
|
||||
|
||||
### Основные настройки
|
||||
```dotenv
|
||||
@ -31,6 +37,25 @@ PACKMATE_MODE=LIVE
|
||||
PACKMATE_INTERFACE=game
|
||||
```
|
||||
|
||||
Для интенсивного трафика можно включить параллельную обработку захваченных пакетов. Оба направления
|
||||
одного TCP/UDP потока всегда обрабатываются одним worker, поэтому порядок пакетов внутри потока
|
||||
сохраняется. Начните с `1` worker как режима совместимости и увеличивайте значение только после
|
||||
сетевого stress test на целевом сервере:
|
||||
|
||||
```dotenv
|
||||
PACKMATE_CAPTURE_WORKERS=1
|
||||
PACKMATE_CAPTURE_QUEUE_CAPACITY=250000
|
||||
PACKMATE_CAPTURE_BUFFER_SIZE_BYTES=67108864
|
||||
PACKMATE_CAPTURE_SHUTDOWN_TIMEOUT_SECONDS=30
|
||||
# Должно быть больше shutdown timeout приложения.
|
||||
PACKMATE_CONTAINER_STOP_GRACE_PERIOD=45s
|
||||
```
|
||||
|
||||
Очередь ограничена: при её переполнении Packmate продолжает работать и увеличивает счётчик
|
||||
`applicationDroppedPackets` в `GET /api/pcap/metrics`. Для production этот счётчик, а также
|
||||
`droppedPackets` и `interfaceDroppedPackets`, должны оставаться равны нулю. `maximumQueuedPackets`
|
||||
показывает максимальное число packet tasks, ожидавших обработки после запуска capture.
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
@ -67,12 +92,15 @@ PACKMATE_MODE=VIEW
|
||||
Для оптимизации работы, рекомендуется включить регулярную очистку БД от старых стримов. Это будет работать только в режиме `LIVE`.
|
||||
```dotenv
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_ENABLED=true
|
||||
# Интервал удаления старых стримов (в минутах).
|
||||
# Лучше ставить маленькое число, чтобы стримы удалялись маленькими кусками, и это не нагружало систему
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL=1
|
||||
# Пауза между запусками cleaner (в секундах).
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL_SECONDS=10
|
||||
# Максимальное время активной очистки за один запуск (в секундах).
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_MAX_RUN_SECONDS=5
|
||||
# Насколько старым стрим должен быть для удаления (в минутах от текущего времени)
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_THRESHOLD=240
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_THRESHOLD=15
|
||||
```
|
||||
Старую переменную `PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL` замените на
|
||||
`PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL_SECONDS`; новая настройка задаётся в секундах.
|
||||
|
||||
### Дополнительные настройки
|
||||
|
||||
@ -86,4 +114,4 @@ BUILD_TAG=latest
|
||||
Чтобы использовать расшифровку TLS (с RSA), нужно положить соответствующий приватный ключ, который
|
||||
использовался для генерации сертификата, в папку `rsa_keys`.
|
||||
|
||||
Файлы БД сохраняются в ./data, поэтому для обнуления базы нужно удалить эту папку.
|
||||
Файлы БД сохраняются в ./data, поэтому для обнуления базы нужно удалить эту папку.
|
||||
|
||||
@ -1,6 +1,12 @@
|
||||
## Setup
|
||||
|
||||
Packmate uses properties from the `.env` file (in the same directory as `docker-compose.yml`)
|
||||
Packmate reads its settings from `.env` in the same directory as `docker-compose.yml`.
|
||||
Create it from the complete template and adjust the values for the server:
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
${EDITOR:-nano} .env
|
||||
```
|
||||
|
||||
### Primary settings
|
||||
```dotenv
|
||||
@ -31,6 +37,24 @@ PACKMATE_MODE=LIVE
|
||||
PACKMATE_INTERFACE=game
|
||||
```
|
||||
|
||||
High traffic installations can process captured packets in parallel. Both directions of one TCP/UDP
|
||||
flow always use the same worker, which preserves packet ordering inside the flow. Start with `1` worker
|
||||
as the compatibility setting and increase it only after running the network stress test on the target host:
|
||||
|
||||
```dotenv
|
||||
PACKMATE_CAPTURE_WORKERS=1
|
||||
PACKMATE_CAPTURE_QUEUE_CAPACITY=250000
|
||||
PACKMATE_CAPTURE_BUFFER_SIZE_BYTES=67108864
|
||||
PACKMATE_CAPTURE_SHUTDOWN_TIMEOUT_SECONDS=30
|
||||
# Must be longer than the application shutdown timeout.
|
||||
PACKMATE_CONTAINER_STOP_GRACE_PERIOD=45s
|
||||
```
|
||||
|
||||
The queue is bounded. If it fills, Packmate remains alive and increments `applicationDroppedPackets`
|
||||
in `GET /api/pcap/metrics`. In production this counter, `droppedPackets`, and
|
||||
`interfaceDroppedPackets` should remain zero. `maximumQueuedPackets` is the maximum number of packet
|
||||
tasks that waited for processing since capture started.
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
@ -67,12 +91,15 @@ On large CTFsб after some time a lot of traffic will pile up. This can slow Pac
|
||||
To optimize the workflow, it is recommended to enable periodical database cleanup of old streams. It will only work in the `LIVE` mode.
|
||||
```dotenv
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_ENABLED=true
|
||||
# Old streams removal interval (in minutes).
|
||||
# It's better to use small numbers so the streams are removed in small chunks and don't overload the server.
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL=1
|
||||
# Delay between cleanup runs (in seconds).
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL_SECONDS=10
|
||||
# Maximum active cleanup time in one run (in seconds).
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_MAX_RUN_SECONDS=5
|
||||
# How old the stream must be to be removed (in minutes before current time)
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_THRESHOLD=240
|
||||
PACKMATE_OLD_STREAMS_CLEANUP_THRESHOLD=15
|
||||
```
|
||||
Replace the old `PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL` variable with
|
||||
`PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL_SECONDS`; the new value is measured in seconds.
|
||||
|
||||
### Additional settings
|
||||
|
||||
@ -85,4 +112,4 @@ BUILD_TAG=latest
|
||||
|
||||
To use the TLS decryption, you have to put the matching private key in the `rsa_keys` directory.
|
||||
|
||||
Database files are being saved in `./data`, so to reset the database, you need to delete this directory.
|
||||
Database files are being saved in `./data`, so to reset the database, you need to delete this directory.
|
||||
|
||||
1
frontend
@ -1 +0,0 @@
|
||||
Subproject commit 9896d4b104cdcfaacd0fec35a3c0e8b8f8ed69cc
|
||||
86
frontend/.gitignore
vendored
Normal file
@ -0,0 +1,86 @@
|
||||
.idea
|
||||
cmake-build-debug/
|
||||
cmake-build-release/
|
||||
*.iws
|
||||
out/
|
||||
.idea_modules/
|
||||
atlassian-ide-plugin.xml
|
||||
com_crashlytics_export_strings.xml
|
||||
crashlytics.properties
|
||||
crashlytics-build.properties
|
||||
fabric.properties
|
||||
*.bak
|
||||
/.tgitconfig
|
||||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
pids
|
||||
*.pid
|
||||
*.seed
|
||||
*.pid.lock
|
||||
lib-cov
|
||||
coverage
|
||||
.nyc_output
|
||||
.grunt
|
||||
bower_components
|
||||
.lock-wscript
|
||||
build/Release
|
||||
node_modules/
|
||||
jspm_packages/
|
||||
typings/
|
||||
.npm
|
||||
.eslintcache
|
||||
.node_repl_history
|
||||
*.tgz
|
||||
.yarn-integrity
|
||||
.env
|
||||
.next
|
||||
Thumbs.db
|
||||
ehthumbs.db
|
||||
ehthumbs_vista.db
|
||||
*.stackdump
|
||||
[Dd]esktop.ini
|
||||
$RECYCLE.BIN/
|
||||
*.cab
|
||||
*.msi
|
||||
*.msix
|
||||
*.msm
|
||||
*.msp
|
||||
*.lnk
|
||||
[._]*.s[a-v][a-z]
|
||||
[._]*.sw[a-p]
|
||||
[._]s[a-v][a-z]
|
||||
[._]sw[a-p]
|
||||
Session.vim
|
||||
.netrwhist
|
||||
*~
|
||||
tags
|
||||
[._]*.un~
|
||||
\#*\#
|
||||
/.emacs.desktop
|
||||
/.emacs.desktop.lock
|
||||
*.elc
|
||||
auto-save-list
|
||||
tramp
|
||||
.\#*
|
||||
.org-id-locations
|
||||
*_archive
|
||||
*_flymake.*
|
||||
/eshell/history
|
||||
/eshell/lastdir
|
||||
/elpa/
|
||||
*.rel
|
||||
/auto/
|
||||
.cask/
|
||||
dist/
|
||||
flycheck_*.el
|
||||
/server/
|
||||
.projectile
|
||||
.dir-locals.el
|
||||
.fuse_hidden*
|
||||
.directory
|
||||
.Trash-*
|
||||
.nfs*
|
||||
$*$
|
||||
29
frontend/README.md
Normal file
@ -0,0 +1,29 @@
|
||||
# packmate-frontend
|
||||
|
||||
## Project setup
|
||||
```
|
||||
npm install
|
||||
```
|
||||
|
||||
### Compiles and hot-reloads for development
|
||||
```
|
||||
npm run serve
|
||||
```
|
||||
|
||||
### Compiles and minifies for production
|
||||
```
|
||||
npm run build
|
||||
```
|
||||
|
||||
### Run your tests
|
||||
```
|
||||
npm run test
|
||||
```
|
||||
|
||||
### Lints and fixes files
|
||||
```
|
||||
npm run lint
|
||||
```
|
||||
|
||||
### Customize configuration
|
||||
See [Configuration Reference](https://cli.vuejs.org/config/).
|
||||
8
frontend/babel.config.js
Normal file
@ -0,0 +1,8 @@
|
||||
module.exports = {
|
||||
presets: [
|
||||
'@vue/app',
|
||||
],
|
||||
plugins: [
|
||||
'@babel/plugin-proposal-optional-chaining',
|
||||
],
|
||||
};
|
||||
31852
frontend/package-lock.json
generated
Normal file
79
frontend/package.json
Normal file
@ -0,0 +1,79 @@
|
||||
{
|
||||
"name": "packmate-frontend",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"serve": "vue-cli-service serve",
|
||||
"build": "vue-cli-service build",
|
||||
"lint": "vue-cli-service lint",
|
||||
"test:ip-patterns": "node --test tests/ip-patterns.test.cjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"axios": "^1.4.0",
|
||||
"bootstrap": "^4.6.2",
|
||||
"bootstrap-darkmode": "^0.9.1",
|
||||
"bootstrap-vue": "^2.23.1",
|
||||
"core-js": "^3.26.1",
|
||||
"idb": "^5.0.8",
|
||||
"sockjs-client": "^1.6.1",
|
||||
"vue": "^2.6.14",
|
||||
"vue-infinite-loading": "^2.4.5",
|
||||
"vue-router": "^3.6.5",
|
||||
"vuex": "^3.6.2",
|
||||
"vuex-persistedstate": "^3.2.1",
|
||||
"vuex-shared-mutations": "^1.0.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/plugin-proposal-optional-chaining": "^7.18.9",
|
||||
"@fortawesome/fontawesome-free": "^5.15.4",
|
||||
"@vue/cli-plugin-babel": "^4.5.19",
|
||||
"@vue/cli-plugin-eslint": "^4.5.19",
|
||||
"@vue/cli-service": "^4.3.0",
|
||||
"babel-eslint": "^10.1.0",
|
||||
"eslint": "^6.8.0",
|
||||
"eslint-plugin-vue": "^6.2.2",
|
||||
"node-sass": "^8.0.0",
|
||||
"sass-loader": "^10.4.1",
|
||||
"vue-template-compiler": "^2.7.14"
|
||||
},
|
||||
"eslintConfig": {
|
||||
"root": true,
|
||||
"env": {
|
||||
"node": true
|
||||
},
|
||||
"extends": [
|
||||
"plugin:vue/essential",
|
||||
"eslint:recommended"
|
||||
],
|
||||
"rules": {
|
||||
"indent": [
|
||||
"off"
|
||||
],
|
||||
"no-mixed-spaces-and-tabs": "off",
|
||||
"comma-dangle": [
|
||||
"warn",
|
||||
{
|
||||
"arrays": "always",
|
||||
"objects": "always",
|
||||
"imports": "always",
|
||||
"exports": "always",
|
||||
"functions": "never"
|
||||
}
|
||||
],
|
||||
"no-console": "off"
|
||||
},
|
||||
"parserOptions": {
|
||||
"parser": "babel-eslint"
|
||||
}
|
||||
},
|
||||
"postcss": {
|
||||
"plugins": {
|
||||
"autoprefixer": {}
|
||||
}
|
||||
},
|
||||
"browserslist": [
|
||||
"> 1%",
|
||||
"last 2 versions",
|
||||
"not ie <= 8"
|
||||
]
|
||||
}
|
||||
BIN
frontend/public/android-chrome-192x192.png
Normal file
|
Before Width: | Height: | Size: 18 KiB After Width: | Height: | Size: 18 KiB |
BIN
frontend/public/android-chrome-512x512.png
Normal file
|
Before Width: | Height: | Size: 54 KiB After Width: | Height: | Size: 54 KiB |
BIN
frontend/public/apple-touch-icon.png
Normal file
|
Before Width: | Height: | Size: 14 KiB After Width: | Height: | Size: 14 KiB |
9
frontend/public/browserconfig.xml
Normal file
@ -0,0 +1,9 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<browserconfig>
|
||||
<msapplication>
|
||||
<tile>
|
||||
<square150x150logo src="/mstile-150x150.png"/>
|
||||
<TileColor>#afd9f0</TileColor>
|
||||
</tile>
|
||||
</msapplication>
|
||||
</browserconfig>
|
||||
BIN
frontend/public/favicon-16x16.png
Normal file
|
Before Width: | Height: | Size: 1.2 KiB After Width: | Height: | Size: 1.2 KiB |
BIN
frontend/public/favicon-32x32.png
Normal file
|
Before Width: | Height: | Size: 2.3 KiB After Width: | Height: | Size: 2.3 KiB |
BIN
frontend/public/favicon.ico
Normal file
|
Before Width: | Height: | Size: 15 KiB After Width: | Height: | Size: 15 KiB |
20
frontend/public/index.html
Normal file
@ -0,0 +1,20 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="ru">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta content="IE=edge" http-equiv="X-UA-Compatible">
|
||||
<title>Packmate</title>
|
||||
|
||||
<meta name="author" content="saber-nyan & serega6531">
|
||||
|
||||
<meta content="width=device-width,initial-scale=1.0" name="viewport">
|
||||
<!--suppress HtmlUnknownTarget -->
|
||||
<link href="<%= BASE_URL %>favicon.ico" rel="icon">
|
||||
</head>
|
||||
<body>
|
||||
<noscript>
|
||||
<strong>Please enable JavaScript to continue!!</strong>
|
||||
</noscript>
|
||||
<div id="app"></div>
|
||||
</body>
|
||||
</html>
|
||||
BIN
frontend/public/mstile-150x150.png
Normal file
|
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 12 KiB |
135
frontend/public/safari-pinned-tab.svg
Normal file
@ -0,0 +1,135 @@
|
||||
<?xml version="1.0" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 20010904//EN"
|
||||
"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
|
||||
<svg version="1.0" xmlns="http://www.w3.org/2000/svg"
|
||||
width="1453.000000pt" height="1453.000000pt" viewBox="0 0 1453.000000 1453.000000"
|
||||
preserveAspectRatio="xMidYMid meet">
|
||||
<metadata>
|
||||
Created by potrace 1.11, written by Peter Selinger 2001-2013
|
||||
</metadata>
|
||||
<g transform="translate(0.000000,1453.000000) scale(0.100000,-0.100000)"
|
||||
fill="#000000" stroke="none">
|
||||
<path d="M7065 14523 c-1372 -40 -2685 -458 -3825 -1216 -157 -105 -420 -294
|
||||
-420 -303 0 -2 17 3 38 11 20 8 84 26 142 40 58 14 148 39 200 54 127 39 219
|
||||
58 314 66 44 4 100 13 124 20 25 8 81 17 126 21 44 3 89 11 99 16 95 49 332
|
||||
73 722 73 306 -1 463 -10 636 -40 167 -28 298 -56 404 -85 44 -12 103 -27 130
|
||||
-34 60 -16 250 -77 320 -104 28 -11 86 -33 130 -50 175 -66 550 -246 619 -296
|
||||
17 -13 67 -44 111 -70 44 -26 102 -61 129 -78 l49 -32 47 17 c88 31 191 13
|
||||
245 -43 l28 -29 51 46 c69 62 286 203 366 238 36 16 70 32 75 36 10 7 190 82
|
||||
290 121 108 42 410 138 433 138 8 0 28 6 45 14 74 32 376 88 657 120 368 43
|
||||
516 44 1020 6 413 -30 709 -73 915 -133 100 -28 208 -97 265 -170 45 -55 45
|
||||
-64 11 -142 -10 -22 -30 -69 -44 -105 -67 -171 -218 -472 -339 -680 -99 -168
|
||||
-127 -225 -128 -254 0 -10 30 -52 68 -94 117 -130 132 -147 132 -154 0 -5 -30
|
||||
-56 -66 -115 -37 -60 -80 -133 -97 -163 l-29 -54 13 -106 c17 -136 15 -322 -4
|
||||
-430 -18 -94 -51 -233 -61 -248 -3 -5 -12 -57 -21 -115 -11 -82 -22 -122 -50
|
||||
-176 -40 -81 -39 -76 -22 -93 12 -12 35 -47 179 -269 59 -90 124 -186 228
|
||||
-337 24 -34 64 -86 89 -115 25 -29 69 -80 96 -112 67 -79 238 -252 298 -299
|
||||
104 -84 198 -144 317 -205 126 -63 185 -100 177 -108 -5 -4 -168 10 -302 27
|
||||
-174 21 -383 102 -532 206 -37 26 -81 57 -98 68 -51 36 -118 98 -163 150 -59
|
||||
71 -58 69 -75 102 -14 25 -16 26 -17 8 0 -12 5 -46 11 -75 6 -30 17 -88 24
|
||||
-129 7 -41 23 -127 35 -190 25 -135 46 -281 60 -435 6 -60 17 -171 25 -245 8
|
||||
-74 17 -227 20 -340 3 -113 10 -234 15 -270 5 -36 12 -361 16 -723 6 -514 4
|
||||
-679 -6 -755 -15 -117 -5 -231 36 -387 14 -55 30 -122 35 -150 5 -27 14 -59
|
||||
19 -70 5 -11 21 -55 35 -97 57 -171 195 -440 297 -582 15 -21 30 -55 34 -75 3
|
||||
-20 19 -49 34 -66 37 -39 59 -94 49 -120 -7 -18 -8 -18 -8 -2 -2 36 -77 134
|
||||
-206 268 -121 125 -211 233 -277 334 -114 176 -160 253 -217 370 -76 158 -129
|
||||
292 -157 400 -6 22 -10 -1 -15 -85 -11 -185 -39 -500 -49 -560 -6 -30 -14 -84
|
||||
-19 -120 -20 -141 -42 -266 -51 -290 -5 -14 -12 -45 -16 -70 -3 -24 -10 -51
|
||||
-14 -60 -4 -8 -15 -40 -24 -70 -8 -30 -24 -72 -35 -93 l-19 -38 59 -112 c32
|
||||
-62 66 -126 74 -144 34 -73 147 -194 252 -270 29 -21 54 -43 56 -48 3 -7 -48
|
||||
-9 -147 -7 -125 3 -164 7 -227 27 -75 23 -179 73 -210 101 -8 8 -20 14 -26 14
|
||||
-24 0 -203 227 -203 258 0 6 -4 12 -8 12 -4 0 -14 16 -21 35 -15 44 -27 45
|
||||
-34 3 -3 -18 -13 -67 -21 -108 -8 -41 -19 -97 -25 -125 -9 -47 -40 -144 -61
|
||||
-190 -49 -111 -60 -139 -60 -151 0 -8 -4 -14 -10 -14 -5 0 -10 -5 -10 -11 0
|
||||
-14 -72 -124 -134 -204 -79 -103 -216 -211 -346 -275 -36 -17 -73 -35 -82 -41
|
||||
-10 -5 -24 -9 -31 -9 -7 0 -29 -11 -47 -25 -20 -15 -36 -21 -38 -15 -4 11 56
|
||||
118 111 199 29 43 161 298 184 356 22 57 113 322 113 331 0 3 -28 -15 -62 -39
|
||||
-93 -66 -223 -138 -308 -171 -41 -15 -84 -32 -95 -37 -18 -7 -178 -49 -189
|
||||
-49 -7 0 -98 -158 -119 -205 -23 -53 -38 -164 -25 -188 5 -10 8 -20 6 -22 -14
|
||||
-14 -74 70 -99 137 -41 111 -64 391 -39 477 5 19 7 37 5 40 -3 2 -59 -51 -126
|
||||
-119 -117 -120 -180 -209 -249 -350 -12 -25 -26 -53 -31 -63 -5 -9 -12 -37
|
||||
-15 -61 l-6 -43 88 -5 c107 -6 186 -41 249 -109 22 -24 80 -88 129 -142 139
|
||||
-152 297 -276 471 -367 44 -23 99 -55 123 -71 24 -16 47 -29 53 -29 5 0 14
|
||||
-11 20 -24 9 -18 26 -28 78 -41 36 -9 104 -28 151 -42 162 -48 179 -52 287
|
||||
-56 153 -6 377 -46 513 -91 54 -17 75 -20 125 -13 108 15 388 22 510 14 66 -4
|
||||
206 -11 310 -15 189 -8 190 -8 260 -42 125 -60 200 -115 256 -187 29 -38 56
|
||||
-72 61 -78 10 -11 305 279 472 465 1000 1111 1631 2489 1815 3965 111 884 56
|
||||
1797 -159 2654 -293 1167 -851 2216 -1656 3111 -124 138 -431 444 -564 561
|
||||
-775 687 -1713 1214 -2692 1514 -605 185 -1213 288 -1858 315 -135 5 -276 9
|
||||
-315 8 -38 -1 -110 -3 -160 -5z"/>
|
||||
<path d="M2499 12743 c-148 -127 -480 -454 -623 -613 -1067 -1189 -1702 -2650
|
||||
-1846 -4242 -43 -480 -29 -1086 35 -1568 198 -1471 822 -2821 1811 -3920 137
|
||||
-152 430 -443 579 -574 106 -94 110 -96 119 -74 21 50 111 196 156 253 42 52
|
||||
61 67 138 104 50 23 114 50 144 59 68 20 327 28 418 11 36 -6 94 -15 130 -20
|
||||
36 -4 124 -18 195 -29 72 -12 158 -25 193 -30 54 -8 152 -34 204 -55 13 -5 21
|
||||
0 29 18 6 14 21 28 33 32 18 5 569 26 764 29 46 1 72 5 72 12 0 7 15 23 33 36
|
||||
54 41 222 138 239 138 8 0 22 7 29 16 7 9 40 37 74 62 58 45 485 466 555 548
|
||||
19 23 67 78 105 124 39 45 80 86 93 91 31 12 84 11 116 -1 23 -8 26 -7 26 14
|
||||
0 19 -27 65 -50 86 -9 8 -72 90 -98 128 -14 21 -42 70 -63 111 l-37 73 -7 -33
|
||||
c-9 -46 -46 -158 -62 -189 -7 -14 -13 -29 -13 -35 0 -18 -44 -98 -93 -172 -81
|
||||
-121 -197 -214 -197 -157 0 13 -67 149 -85 172 -92 115 -132 161 -139 161 -5
|
||||
1 -37 27 -71 58 -35 32 -69 63 -76 68 -24 18 -86 55 -93 55 -7 0 46 -114 57
|
||||
-118 4 -2 7 -8 7 -14 0 -5 17 -40 39 -76 105 -182 127 -232 101 -232 -5 0 -10
|
||||
4 -10 8 0 5 -10 14 -22 19 -62 29 -319 251 -360 310 -16 24 -37 51 -47 61 -21
|
||||
21 -66 79 -113 147 -28 42 -103 193 -145 295 -8 21 -11 14 -16 -48 -8 -79 -45
|
||||
-261 -57 -277 -4 -5 -11 -26 -15 -45 -4 -19 -11 -39 -15 -45 -4 -5 -13 -28
|
||||
-20 -50 -35 -108 -220 -355 -266 -355 -13 0 -26 -4 -29 -10 -3 -5 -34 -26 -68
|
||||
-46 -70 -41 -77 -48 -77 -87 0 -16 -4 -26 -10 -22 -15 9 -12 73 3 82 40 24
|
||||
110 216 126 343 6 41 15 106 21 143 10 58 9 71 -3 85 -18 20 -127 181 -127
|
||||
187 0 2 -13 23 -29 47 -16 24 -42 66 -57 94 -16 28 -60 104 -99 168 -111 186
|
||||
-308 565 -330 636 -4 14 -10 27 -14 30 -3 3 -14 21 -23 40 l-17 35 0 -60 c0
|
||||
-107 60 -452 88 -505 5 -9 12 -31 16 -50 4 -19 11 -39 15 -45 4 -5 11 -26 15
|
||||
-45 19 -90 158 -362 271 -530 48 -71 85 -132 83 -135 -6 -5 -145 86 -241 158
|
||||
-128 97 -297 286 -298 335 0 6 -3 12 -8 12 -7 0 -38 56 -65 120 -9 19 -25 57
|
||||
-36 83 -11 26 -23 63 -27 83 -3 19 -10 36 -14 39 -5 3 -11 24 -15 48 -4 23
|
||||
-11 47 -16 53 -4 5 -13 44 -19 85 -6 40 -17 83 -23 94 -16 28 -22 822 -7 965
|
||||
6 58 16 164 21 235 6 72 17 153 24 180 20 73 28 132 17 120 -5 -5 -23 -35 -39
|
||||
-65 -50 -92 -58 -105 -70 -113 -7 -4 -13 -12 -13 -18 0 -13 -41 -72 -58 -82
|
||||
-6 -4 -12 -14 -12 -20 0 -17 -188 -217 -204 -217 -7 0 -24 -10 -37 -21 -45
|
||||
-41 -214 -145 -259 -161 -25 -9 -76 -19 -114 -22 -38 -4 -82 -14 -98 -22 -16
|
||||
-8 -33 -12 -38 -9 -15 9 16 36 74 66 70 35 96 51 118 72 10 9 23 17 28 17 6 0
|
||||
10 5 10 11 0 7 11 22 24 34 42 39 114 152 167 263 65 137 63 131 95 237 15 50
|
||||
30 95 35 100 4 6 10 28 14 50 4 22 13 46 19 54 14 18 13 241 -2 250 -5 3 -13
|
||||
38 -17 76 -3 39 -10 79 -15 90 -5 11 -14 67 -20 125 -6 58 -15 140 -20 183
|
||||
-45 362 -43 1174 4 1477 8 52 18 124 22 160 3 35 10 71 15 80 4 8 12 44 18 80
|
||||
12 83 66 302 80 330 5 9 12 31 16 50 4 19 13 46 21 60 7 14 31 70 54 125 22
|
||||
55 45 108 50 117 19 34 9 36 -47 8 -117 -57 -191 -86 -263 -101 -25 -6 -54
|
||||
-14 -65 -20 -58 -27 -335 -42 -460 -24 -80 11 -194 72 -302 163 -80 66 -71 73
|
||||
47 42 56 -15 316 -8 333 9 6 6 24 11 40 11 16 0 43 6 60 14 18 8 61 27 97 42
|
||||
36 16 83 39 105 51 22 13 51 28 65 35 14 7 27 15 30 18 3 3 16 11 30 18 48 24
|
||||
70 39 126 83 31 24 90 71 132 104 43 33 82 64 89 70 6 5 28 21 47 35 42 29 42
|
||||
45 3 113 -42 75 -57 115 -76 212 -34 173 -2 389 109 725 27 82 60 213 60 237
|
||||
0 6 -20 35 -44 63 -48 56 -58 90 -44 156 11 48 32 11 -207 359 -126 183 -363
|
||||
575 -408 675 -13 30 -36 80 -50 110 -60 129 -66 151 -52 198 7 23 12 42 10 42
|
||||
-1 0 -54 -44 -116 -97z"/>
|
||||
<path d="M6445 1929 c-5 -7 -25 -8 -56 -4 -29 4 -49 3 -49 -3 0 -5 -35 -12
|
||||
-77 -16 -80 -6 -212 -31 -248 -46 -11 -4 -36 -14 -55 -21 -19 -7 -53 -24 -75
|
||||
-39 -46 -30 -58 -37 -142 -86 -35 -19 -65 -40 -69 -45 -3 -5 13 -48 36 -97 62
|
||||
-131 68 -143 90 -200 11 -29 20 -62 20 -73 0 -11 16 -47 35 -80 19 -33 38 -72
|
||||
41 -87 4 -15 10 -31 14 -37 14 -18 32 -174 28 -240 -3 -54 -1 -65 12 -65 15 0
|
||||
145 40 190 58 14 6 59 20 100 32 41 12 84 25 95 30 51 20 248 80 266 80 11 0
|
||||
52 14 91 30 107 46 113 40 38 -35 -27 -28 -50 -54 -50 -58 0 -4 -46 -54 -102
|
||||
-110 -104 -103 -238 -259 -238 -276 0 -5 -3 -11 -8 -13 -7 -3 -27 -30 -133
|
||||
-177 -18 -24 -49 -76 -69 -115 -20 -39 -44 -84 -54 -99 -10 -15 -17 -29 -15
|
||||
-32 13 -13 474 -67 765 -89 l70 -6 11 68 c15 90 43 205 54 216 5 6 9 17 9 26
|
||||
0 23 103 221 159 305 25 38 61 99 79 135 18 35 40 74 50 85 29 35 132 252 132
|
||||
279 0 9 4 16 9 16 5 0 14 21 20 46 10 40 74 126 84 113 10 -12 27 -70 32 -109
|
||||
7 -46 38 -152 60 -205 7 -16 16 -41 20 -55 3 -14 12 -38 20 -55 7 -16 18 -41
|
||||
23 -55 24 -55 75 -158 105 -211 35 -62 137 -221 147 -229 3 -3 20 -34 37 -70
|
||||
18 -36 48 -88 67 -117 19 -29 37 -65 41 -81 3 -17 14 -37 24 -46 17 -15 29
|
||||
-15 167 4 182 25 536 89 547 100 7 7 -67 111 -85 118 -4 2 -8 8 -8 14 0 11
|
||||
-88 121 -101 126 -5 2 -9 8 -9 13 0 5 -19 33 -42 62 -24 29 -54 68 -68 86 -14
|
||||
18 -34 45 -46 59 -32 41 -124 171 -124 176 0 3 -19 33 -43 68 -24 35 -53 82
|
||||
-65 106 l-22 42 28 0 c15 -1 54 -7 87 -15 33 -8 83 -19 110 -25 28 -6 57 -15
|
||||
66 -20 9 -5 33 -12 53 -15 20 -4 53 -16 73 -26 21 -11 48 -19 60 -19 13 0 23
|
||||
-4 23 -9 0 -5 13 -11 29 -14 16 -4 56 -19 89 -35 l60 -28 6 60 c3 32 6 85 6
|
||||
117 0 33 4 59 9 59 4 0 11 25 14 56 3 31 11 62 16 67 6 6 11 19 11 29 0 10 4
|
||||
18 8 18 4 0 15 28 25 63 10 34 49 124 87 199 72 141 83 184 53 195 -10 4 -37
|
||||
19 -60 35 -23 15 -45 28 -49 28 -8 0 -22 6 -169 75 -44 21 -100 40 -125 43
|
||||
-25 2 -50 8 -55 13 -6 4 -51 13 -100 19 -50 7 -100 17 -113 23 -26 12 -340 -5
|
||||
-407 -23 -100 -26 -153 -44 -194 -62 -38 -18 -55 -20 -146 -14 -56 3 -127 13
|
||||
-157 21 -30 7 -74 12 -99 10 -99 -8 -216 -28 -246 -42 -29 -13 -37 -13 -75 0
|
||||
-64 22 -148 45 -193 53 -22 3 -51 13 -64 21 -14 9 -59 18 -108 21 -46 3 -86 9
|
||||
-89 13 -8 13 -192 11 -199 -2z"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 9.3 KiB After Width: | Height: | Size: 9.3 KiB |
21
frontend/public/site.webmanifest
Normal file
@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "Packmate",
|
||||
"short_name": "Packmate",
|
||||
"start_url": ".",
|
||||
"description": "Packet monitoring tool for CTFs.",
|
||||
"icons": [
|
||||
{
|
||||
"src": "/android-chrome-192x192.png",
|
||||
"sizes": "192x192",
|
||||
"type": "image/png"
|
||||
},
|
||||
{
|
||||
"src": "/android-chrome-512x512.png",
|
||||
"sizes": "512x512",
|
||||
"type": "image/png"
|
||||
}
|
||||
],
|
||||
"theme_color": "#afd9f0",
|
||||
"background_color": "#ffffff",
|
||||
"display": "standalone"
|
||||
}
|
||||
314
frontend/src/App.vue
Normal file
@ -0,0 +1,314 @@
|
||||
<template>
|
||||
<div id="app">
|
||||
<Navbar/>
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
<transition name="fade" mode="out-in">
|
||||
<router-view name="sidebar" ref="sidebar"/>
|
||||
</transition>
|
||||
<main role="main" class="col-sm-9 ml-sm-auto px-4">
|
||||
<transition name="fade" mode="out-in">
|
||||
<router-view name="content"/>
|
||||
</transition>
|
||||
</main>
|
||||
</div>
|
||||
</div>
|
||||
<Settings/>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<!--suppress JSUnresolvedVariable -->
|
||||
<script>
|
||||
export const DB_OBJSTORE_COUNTERS_HISTORY = 'countersHistory';
|
||||
|
||||
import Navbar from './components/Navbar';
|
||||
import Settings from './views/Settings';
|
||||
import SockJS from 'sockjs-client';
|
||||
import {openDB,} from 'idb';
|
||||
|
||||
export default {
|
||||
data() {
|
||||
return {
|
||||
websocket: null,
|
||||
db: null,
|
||||
};
|
||||
},
|
||||
mounted() {
|
||||
this.connectWs();
|
||||
openDB('packmate', 1, {
|
||||
upgrade(db, oldVersion, newVersion) {
|
||||
console.info('[IDB] Creating new database! Old rev %d, new rev %d',
|
||||
oldVersion, newVersion);
|
||||
db.createObjectStore(DB_OBJSTORE_COUNTERS_HISTORY, {
|
||||
autoIncrement: false,
|
||||
keyPath: null,
|
||||
});
|
||||
},
|
||||
})
|
||||
.then(db => {
|
||||
this.db = db;
|
||||
})
|
||||
.catch(e => {
|
||||
console.error('[IDB] Failed to open DB!', e);
|
||||
});
|
||||
},
|
||||
beforeDestroy() {
|
||||
this.websocket?.close();
|
||||
},
|
||||
methods: {
|
||||
connectWs() {
|
||||
if (this.websocket !== null) return;
|
||||
this.websocket = new SockJS(this.$http.defaults.baseURL + '/ws');
|
||||
this.websocket.onopen = () => {
|
||||
console.info('[WS] Connected');
|
||||
this.$nextTick(() => this.$refs.sidebar?.scheduleReconciliation(0));
|
||||
};
|
||||
this.websocket.onclose = (ev) => {
|
||||
console.info('[WS] Disconnected', ev.code, ev.reason);
|
||||
this.websocket = null;
|
||||
if (ev.code === 1008) {
|
||||
console.info('[WS] Security timeout, reconnecting...');
|
||||
this.connectWs();
|
||||
}
|
||||
if (ev.code !== 1000) { // Normal closure
|
||||
setTimeout(this.connectWs, 3000);
|
||||
console.info('[WS] Reconnecting...');
|
||||
}
|
||||
};
|
||||
this.websocket.onmessage = (ev) => {
|
||||
const parsed = JSON.parse(ev.data);
|
||||
|
||||
switch (parsed.type) {
|
||||
case 'NEW_STREAM': {
|
||||
this.$refs.sidebar.addStreamFromWs(parsed.value);
|
||||
break;
|
||||
}
|
||||
case 'STREAMS_INVALIDATED': {
|
||||
this.$refs.sidebar.scheduleReconciliation();
|
||||
break;
|
||||
}
|
||||
case 'SAVE_SERVICE': {
|
||||
this.addServiceFromWs(parsed.value);
|
||||
break;
|
||||
}
|
||||
case 'DELETE_SERVICE': {
|
||||
this.deleteServiceFromWs(parsed.value);
|
||||
break;
|
||||
}
|
||||
case 'SAVE_PATTERN': {
|
||||
this.addPatternFromWs(parsed.value);
|
||||
break;
|
||||
}
|
||||
case 'COUNTERS_UPDATE': {
|
||||
const data = parsed.value;
|
||||
this.$store.commit('setCurrentPacketsCount', data.totalPackets);
|
||||
this.$store.commit('setCurrentStreamsCount', data.totalStreams);
|
||||
this.$store.commit('setCurrentServicesPacketsCount', data.servicesPackets);
|
||||
this.$store.commit('setCurrentServicesStreamsCount', data.servicesStreams);
|
||||
|
||||
console.debug('Adding new counters to DB', parsed.value);
|
||||
const tx = this.db.transaction(DB_OBJSTORE_COUNTERS_HISTORY, 'readwrite');
|
||||
tx.store.add({
|
||||
newPacketsCount: data.totalPackets,
|
||||
newStreamsCount: data.totalStreams,
|
||||
servicesPackets: data.servicesPackets,
|
||||
servicesStreams: data.servicesStreams,
|
||||
}, Date.now()).then(() => {
|
||||
console.debug('[IDB] Added entry');
|
||||
}).catch(e => {
|
||||
console.error('[IDB] Failed to add entry!', e);
|
||||
});
|
||||
break;
|
||||
}
|
||||
case 'ENABLE_PATTERN': {
|
||||
this.togglePatternFromWs(parsed.value, true);
|
||||
break;
|
||||
}
|
||||
case 'DISABLE_PATTERN': {
|
||||
this.togglePatternFromWs(parsed.value, false);
|
||||
break;
|
||||
}
|
||||
case 'PCAP_STARTED': {
|
||||
this.$store.commit('startPcap');
|
||||
this.$bvToast.toast(`Pcap file processing started`, {
|
||||
title: 'Notification',
|
||||
variant: 'info',
|
||||
autoHideDelay: 5000,
|
||||
});
|
||||
break;
|
||||
}
|
||||
case 'PCAP_STOPPED': {
|
||||
this.$bvToast.toast(`All streams processed`, {
|
||||
title: 'Notification',
|
||||
variant: 'success',
|
||||
autoHideDelay: 5000,
|
||||
});
|
||||
break;
|
||||
}
|
||||
case 'FINISH_LOOKBACK': {
|
||||
console.debug('Lookback completed');
|
||||
this.$bvToast.toast(`Lookback completed`, {
|
||||
title: 'Notification',
|
||||
variant: 'success',
|
||||
autoHideDelay: 5000,
|
||||
});
|
||||
this.$refs.sidebar.streams = [];
|
||||
this.$refs.sidebar.$refs.infiniteLoader.stateChanger.reset();
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
console.error('[WS] Event is not implemented!', parsed);
|
||||
break;
|
||||
}
|
||||
}
|
||||
};
|
||||
this.websocket.onerror = (ev) => {
|
||||
console.warn('[WS] Error', ev);
|
||||
};
|
||||
},
|
||||
addPatternFromWs(pattern) {
|
||||
const foundIndex = this.$store.state.patterns.findIndex(el => el.id === pattern.id);
|
||||
if (foundIndex === -1) {
|
||||
this.$store.commit('addPattern', pattern);
|
||||
return;
|
||||
}
|
||||
|
||||
let newPatterns = this.$store.state.patterns.slice();
|
||||
newPatterns.splice(foundIndex, 1, pattern);
|
||||
this.$store.commit('setPatterns', newPatterns);
|
||||
},
|
||||
togglePatternFromWs(id, enabled) {
|
||||
this.$store.state.patterns.forEach(pattern => {
|
||||
if (pattern.id === id) {
|
||||
pattern.enabled = enabled;
|
||||
}
|
||||
});
|
||||
},
|
||||
addServiceFromWs(service) {
|
||||
const foundIndex = this.$store.state.services.findIndex(el => el.port === service.port);
|
||||
if (foundIndex === -1) {
|
||||
this.$store.commit('addService', service);
|
||||
return;
|
||||
}
|
||||
|
||||
let newServices = this.$store.state.services.slice();
|
||||
newServices.splice(foundIndex, 1, service);
|
||||
this.$store.commit('setServices', newServices);
|
||||
},
|
||||
deleteServiceFromWs(port) {
|
||||
this.$store.commit('setServices', this.$store.state.services.filter(o => o.port !== port));
|
||||
},
|
||||
},
|
||||
components: {
|
||||
Settings,
|
||||
Navbar,
|
||||
},
|
||||
};
|
||||
</script>
|
||||
|
||||
<style>
|
||||
/* cyrillic-ext */
|
||||
@font-face {
|
||||
font-family: 'Open Sans';
|
||||
font-style: normal;
|
||||
font-weight: 400;
|
||||
font-display: swap;
|
||||
src: local('Open Sans Regular'), local('OpenSans-Regular'), url(https://fonts.gstatic.com/s/opensans/v16/mem8YaGs126MiZpBA-UFWJ0bbck.woff2) format('woff2');
|
||||
unicode-range: U+0460-052F, U+1C80-1C88, U+20B4, U+2DE0-2DFF, U+A640-A69F, U+FE2E-FE2F;
|
||||
}
|
||||
|
||||
/* cyrillic */
|
||||
@font-face {
|
||||
font-family: 'Open Sans';
|
||||
font-style: normal;
|
||||
font-weight: 400;
|
||||
font-display: swap;
|
||||
src: local('Open Sans Regular'), local('OpenSans-Regular'), url(https://fonts.gstatic.com/s/opensans/v16/mem8YaGs126MiZpBA-UFUZ0bbck.woff2) format('woff2');
|
||||
unicode-range: U+0400-045F, U+0490-0491, U+04B0-04B1, U+2116;
|
||||
}
|
||||
|
||||
/* greek-ext */
|
||||
@font-face {
|
||||
font-family: 'Open Sans';
|
||||
font-style: normal;
|
||||
font-weight: 400;
|
||||
font-display: swap;
|
||||
src: local('Open Sans Regular'), local('OpenSans-Regular'), url(https://fonts.gstatic.com/s/opensans/v16/mem8YaGs126MiZpBA-UFWZ0bbck.woff2) format('woff2');
|
||||
unicode-range: U+1F00-1FFF;
|
||||
}
|
||||
|
||||
/* greek */
|
||||
@font-face {
|
||||
font-family: 'Open Sans';
|
||||
font-style: normal;
|
||||
font-weight: 400;
|
||||
font-display: swap;
|
||||
src: local('Open Sans Regular'), local('OpenSans-Regular'), url(https://fonts.gstatic.com/s/opensans/v16/mem8YaGs126MiZpBA-UFVp0bbck.woff2) format('woff2');
|
||||
unicode-range: U+0370-03FF;
|
||||
}
|
||||
|
||||
/* vietnamese */
|
||||
@font-face {
|
||||
font-family: 'Open Sans';
|
||||
font-style: normal;
|
||||
font-weight: 400;
|
||||
font-display: swap;
|
||||
src: local('Open Sans Regular'), local('OpenSans-Regular'), url(https://fonts.gstatic.com/s/opensans/v16/mem8YaGs126MiZpBA-UFWp0bbck.woff2) format('woff2');
|
||||
unicode-range: U+0102-0103, U+0110-0111, U+1EA0-1EF9, U+20AB;
|
||||
}
|
||||
|
||||
/* latin-ext */
|
||||
@font-face {
|
||||
font-family: 'Open Sans';
|
||||
font-style: normal;
|
||||
font-weight: 400;
|
||||
font-display: swap;
|
||||
src: local('Open Sans Regular'), local('OpenSans-Regular'), url(https://fonts.gstatic.com/s/opensans/v16/mem8YaGs126MiZpBA-UFW50bbck.woff2) format('woff2');
|
||||
unicode-range: U+0100-024F, U+0259, U+1E00-1EFF, U+2020, U+20A0-20AB, U+20AD-20CF, U+2113, U+2C60-2C7F, U+A720-A7FF;
|
||||
}
|
||||
|
||||
/* latin */
|
||||
@font-face {
|
||||
font-family: 'Open Sans';
|
||||
font-style: normal;
|
||||
font-weight: 400;
|
||||
font-display: swap;
|
||||
src: local('Open Sans Regular'), local('OpenSans-Regular'), url(https://fonts.gstatic.com/s/opensans/v16/mem8YaGs126MiZpBA-UFVZ0b.woff2) format('woff2');
|
||||
unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+2000-206F, U+2074, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD;
|
||||
}
|
||||
|
||||
html * {
|
||||
font-family: 'Open Sans', Helvetica, Arial, sans-serif;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
-moz-osx-font-smoothing: grayscale;
|
||||
font-size: .875rem;
|
||||
}
|
||||
|
||||
body {
|
||||
overflow-y: scroll;
|
||||
}
|
||||
|
||||
/*noinspection CssUnusedSymbol*/
|
||||
.fade-enter-active, .fade-leave-active {
|
||||
transition: opacity .3s;
|
||||
}
|
||||
|
||||
/*noinspection CssUnusedSymbol*/
|
||||
.fade-enter, .fade-leave-to {
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
/* Patterns dropdown fix */
|
||||
/*noinspection CssUnusedSymbol*/
|
||||
.dropdown-menu.show {
|
||||
position: fixed !important;
|
||||
/*transform: translate3d(15px, 38px, 0) !important;*/
|
||||
transform: none !important;
|
||||
left: 15px !important;
|
||||
top: 38px !important;
|
||||
}
|
||||
|
||||
[role="main"] {
|
||||
padding-top: 55px; /* Space for fixed navbar */
|
||||
}
|
||||
</style>
|
||||
182
frontend/src/components/Navbar.vue
Normal file
@ -0,0 +1,182 @@
|
||||
<template>
|
||||
<nav class="navbar navbar-dark navbar-expand fixed-top bg-dark flex-md-nowrap p-0 shadow">
|
||||
<span class="navbar-brand mb-0 ml-2">Packmate</span>
|
||||
<span class="navbar-text navbar-nowrap">
|
||||
{{ this.$store.state.currentStreamsCount }}
|
||||
<u title="Streams per minute">SPM</u>
|
||||
, {{ packetsPerStream }}
|
||||
<u title="Packets per stream (average)">PPS</u>
|
||||
</span>
|
||||
|
||||
<PatternsDropdown ref="patternsDropdown"/>
|
||||
|
||||
<span v-if="this.$route.query.pattern" class="navbar-text navbar-nowrap">
|
||||
{{ selectedPatternText }}
|
||||
</span>
|
||||
|
||||
<div class="navbar-collapse collapse">
|
||||
<ul class="navbar-nav px-1 mr-auto">
|
||||
<li class="nav-item text-nowrap">
|
||||
<router-link class="nav-link" :to="{name:'stream', params: {}, query: $route.query}" exact>All</router-link>
|
||||
</li>
|
||||
<template v-for="service in this.$store.state.services">
|
||||
<router-link :key="service.port" tag="li" class="nav-item text-nowrap edit-button"
|
||||
:to="{name:'stream', params: {servicePort: service.port}, query: $route.query}">
|
||||
<a class="nav-link">
|
||||
{{service.name}} #{{service.port}}
|
||||
({{ getSpmForService(service.port) }}
|
||||
<u title="Streams per minute">SPM</u>)
|
||||
</a>
|
||||
|
||||
<a class="nav-link pl-0" style="cursor: pointer" @click.stop.prevent="editService(service)">
|
||||
<i class="fas fa-pencil-alt"/>
|
||||
</a>
|
||||
</router-link>
|
||||
</template>
|
||||
<li class="nav-item text-nowrap" style="padding-left: 1em;">
|
||||
<div class="my-2 mr-3 navbar-cogs" style="cursor: pointer;"
|
||||
@click.stop.prevent="showAddService">
|
||||
<i class="fas fa-plus-circle"/>
|
||||
</div>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
<div class="my-2 my-lg-0 mr-3 navbar-cogs" style="cursor: pointer;"
|
||||
@click.stop.prevent="showSettings">
|
||||
<i class="fas fa-cogs"/>
|
||||
</div>
|
||||
</div>
|
||||
<ServiceModal :creating="serviceModalIsCreating" :initial-service="serviceModalEditingService"
|
||||
@service-update-needed="updateServices"/>
|
||||
</nav>
|
||||
</template>
|
||||
|
||||
<script>
|
||||
import PatternsDropdown from './PatternsDropdown';
|
||||
import ServiceModal from '../views/ServiceModal';
|
||||
|
||||
export default {
|
||||
name: 'Navbar',
|
||||
computed: {
|
||||
packetsPerStream: function() {
|
||||
let streams = this.$store.state.currentStreamsCount;
|
||||
let packets = this.$store.state.currentPacketsCount;
|
||||
|
||||
if (streams === 0) {
|
||||
return 0;
|
||||
} else {
|
||||
let pps = packets / streams;
|
||||
return Math.round((pps + Number.EPSILON) * 10) / 10;
|
||||
}
|
||||
},
|
||||
selectedPatternText: function () {
|
||||
let selected = this.$route.query.pattern;
|
||||
if (typeof selected === 'string') {
|
||||
selected = parseInt(selected);
|
||||
}
|
||||
|
||||
let pattern = this.$store.state.patterns.find(o => o.id === selected);
|
||||
if (pattern) {
|
||||
return `[Selected: ${pattern.name}]`;
|
||||
} else {
|
||||
return '[Invalid pattern]';
|
||||
}
|
||||
},
|
||||
},
|
||||
data() {
|
||||
return {
|
||||
serviceModalIsCreating: true,
|
||||
serviceModalEditingService: {},
|
||||
};
|
||||
},
|
||||
mounted() {
|
||||
this.updateServices();
|
||||
},
|
||||
methods: {
|
||||
getSpmForService(port) {
|
||||
return this.$store.state.currentServicesStreamsCount[port] ?? 0;
|
||||
},
|
||||
editService(service) {
|
||||
this.serviceModalIsCreating = false;
|
||||
this.serviceModalEditingService = {};
|
||||
this.$nextTick(() => {
|
||||
this.serviceModalEditingService = service;
|
||||
this.$bvModal.show('serviceModal');
|
||||
});
|
||||
},
|
||||
showSettings() {
|
||||
this.$bvModal.show('settingsModal');
|
||||
console.debug('Showing settings...');
|
||||
},
|
||||
showAddService() {
|
||||
this.serviceModalIsCreating = true;
|
||||
this.serviceModalEditingService = {};
|
||||
this.$bvModal.show('serviceModal');
|
||||
console.debug('Showing addService...');
|
||||
},
|
||||
updateServices() {
|
||||
this.$http.get('service/')
|
||||
.then(r => this.$store.commit('setServices', r.data))
|
||||
.catch(e => {
|
||||
this.$bvToast.toast(`Failed to load services: ${e}`, {
|
||||
title: 'Error',
|
||||
variant: 'danger',
|
||||
});
|
||||
console.error('Failed to load services:', e);
|
||||
});
|
||||
},
|
||||
},
|
||||
components: {
|
||||
ServiceModal,
|
||||
PatternsDropdown,
|
||||
},
|
||||
};
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.nav-link {
|
||||
transition: all .3s;
|
||||
}
|
||||
|
||||
.edit-button {
|
||||
display: inherit;
|
||||
}
|
||||
|
||||
.navbar-cogs > i {
|
||||
transition: all .3s;
|
||||
color: rgba(255, 255, 255, 0.7);
|
||||
}
|
||||
|
||||
.navbar-cogs > i:hover {
|
||||
color: #FFF;
|
||||
}
|
||||
|
||||
.navbar-nowrap {
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
nav {
|
||||
overflow-x: auto;
|
||||
}
|
||||
|
||||
u {
|
||||
text-underline-position: under;
|
||||
text-decoration-style: dotted;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar {
|
||||
height: 0.2em
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-button {
|
||||
width: 0;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-track-piece {
|
||||
background: #F1F1F1
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-thumb {
|
||||
background: #C1C1C1
|
||||
}
|
||||
</style>
|
||||
226
frontend/src/components/Packet.vue
Normal file
@ -0,0 +1,226 @@
|
||||
<template>
|
||||
<div :class="{'packet-incoming': packet.incoming, 'packet-outgoing': !packet.incoming}">
|
||||
<div>#{{ packet.id }} at {{ dateToText(packet.timestamp) }}
|
||||
<template v-if="offset !== null"> (+{{ offset }} ms)</template>
|
||||
{{ printPacketFlags(packet) }}
|
||||
<button @click.prevent="copyRaw" class="btn btn-link">Copy HEX</button>
|
||||
<button @click.prevent="copyText" class="btn btn-link">Copy text</button>
|
||||
<button @click.prevent="copyPythonBytes" class="btn btn-link">Copy as Python bytes</button>
|
||||
</div>
|
||||
<div v-if="packet.sourceIp || packet.destinationIp" class="packet-addresses">
|
||||
<span title="Source IP" :style="ipHighlightStyle('SOURCE_IP')">{{ packet.sourceIp || '?' }}</span>
|
||||
→
|
||||
<span title="Destination IP" :style="ipHighlightStyle('DESTINATION_IP')">{{ packet.destinationIp || '?' }}</span>
|
||||
</div>
|
||||
<p v-if="!this.$store.state.hexdumpMode"
|
||||
class="pt-2 pb-2 mb-3"
|
||||
v-html="stringdata"/>
|
||||
<p v-else
|
||||
class="pt-2 pb-2 mb-3"
|
||||
v-html="hexdata"/>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<!--suppress JSUnresolvedVariable, JSDeprecatedSymbols -->
|
||||
<script>
|
||||
export default {
|
||||
name: 'Packet',
|
||||
props: {
|
||||
packet: {
|
||||
id: Number(),
|
||||
matches: Array(),
|
||||
timestamp: Number(),
|
||||
sourceIp: String(),
|
||||
destinationIp: String(),
|
||||
incoming: Boolean(),
|
||||
ungzipped: Boolean(),
|
||||
webSocketParsed: Boolean(),
|
||||
tlsDecrypted: Boolean(),
|
||||
content: String(),
|
||||
},
|
||||
offset: Number(),
|
||||
},
|
||||
computed: {
|
||||
hexdata() {
|
||||
const dataString = this.atou(this.packet.content);
|
||||
const dump = this.hexdump(dataString, this.$store.state.hexdumpBlockSize, this.$store.state.hexdumpLineNumberBase);
|
||||
return this.escapeHtml(dump)
|
||||
.split('\n')
|
||||
.join('<br>'); // Replace all \n to <br>
|
||||
},
|
||||
stringdata() {
|
||||
const dataString = this.atou(this.packet.content);
|
||||
const dump = this.highlightPatterns(dataString);
|
||||
return this.escapeHtml(dump)
|
||||
.split('\n')
|
||||
.join('<br>');
|
||||
},
|
||||
},
|
||||
methods: {
|
||||
ipHighlightStyle(target) {
|
||||
const match = (this.packet.matches || []).find(match => match.searchTarget === target &&
|
||||
this.$store.state.patterns.some(pattern => pattern.id === match.patternId && !pattern.deleted));
|
||||
if (!match) return {};
|
||||
const pattern = this.$store.state.patterns.find(pattern => pattern.id === match.patternId);
|
||||
return {backgroundColor: pattern.color,};
|
||||
},
|
||||
atou(b64) {
|
||||
const text = atob(b64);
|
||||
const length = text.length;
|
||||
const bytes = new Uint8Array(length);
|
||||
for (let i = 0; i < length; i++) {
|
||||
bytes[i] = text.charCodeAt(i);
|
||||
}
|
||||
const decoder = new TextDecoder();
|
||||
return decoder.decode(bytes);
|
||||
},
|
||||
printPacketFlags(packet) {
|
||||
let flags = [];
|
||||
|
||||
if (packet.ungzipped) {
|
||||
flags.push('GZIP');
|
||||
}
|
||||
|
||||
if (packet.webSocketParsed) {
|
||||
flags.push('WS');
|
||||
}
|
||||
|
||||
if (packet.tlsDecrypted) {
|
||||
flags.push('TLS');
|
||||
}
|
||||
|
||||
return flags.join(' ');
|
||||
},
|
||||
hexdump(buffer, blockSize, lineNumberBase) {
|
||||
blockSize = parseInt(blockSize, 10) || 16;
|
||||
lineNumberBase = parseInt(lineNumberBase, 10) || 10;
|
||||
let lines = [];
|
||||
const hex = '0123456789ABCDEF';
|
||||
for (let b = 0; b < buffer.length; b += blockSize) {
|
||||
const block = buffer.slice(b, Math.min(b + blockSize, buffer.length));
|
||||
const addr = ('0000000000' + b.toString(lineNumberBase)).slice(-10);
|
||||
let codes = block.split('').map(ch => {
|
||||
const code = ch.charCodeAt(0);
|
||||
return ' ' + hex[(0xF0 & code) >> 4] + hex[0x0F & code];
|
||||
}).join('');
|
||||
codes += ' ..'.repeat(blockSize - block.length);
|
||||
// eslint-disable-next-line no-control-regex
|
||||
let chars = block.replace(/[\x00-\x1F]/g, '.');
|
||||
chars += ' '.repeat(blockSize - block.length);
|
||||
lines.push(addr + ':' + codes + ' |' + chars + '|');
|
||||
}
|
||||
return lines.join('\n');
|
||||
},
|
||||
dateToText(unixTimestamp) {
|
||||
return new Date(unixTimestamp).toLocaleDateString('ru-RU', {
|
||||
month: '2-digit',
|
||||
day: '2-digit',
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
second: '2-digit',
|
||||
});
|
||||
},
|
||||
escapeHtml(in_) {
|
||||
return in_.replace(/(<span style="background-color: #(?:[A-Fa-f0-9]{6}|[A-Fa-f0-9]{3})">|<\/span>)|[&<>"'/]/g, ($0, $1) => {
|
||||
const entityMap = {
|
||||
'&': '&',
|
||||
'<': '<',
|
||||
'>': '>',
|
||||
'"': '"',
|
||||
'\'': ''',
|
||||
'/': '/',
|
||||
};
|
||||
|
||||
return $1 ? $1 : entityMap[$0];
|
||||
});
|
||||
},
|
||||
highlightPatterns(raw) {
|
||||
const patterns = this.$store.state.patterns.reduce((obj, item) => {
|
||||
obj[item.id] = item;
|
||||
return obj;
|
||||
}, {}); // Array to object
|
||||
let offset = 0;
|
||||
this.packet.matches
|
||||
.filter(match => !match.searchTarget || match.searchTarget === 'CONTENT')
|
||||
.filter(match => !patterns[match.patternId].deleted)
|
||||
.sort((a, b) => a.startPosition - b.startPosition)
|
||||
.forEach(match => {
|
||||
const pattern = patterns[match.patternId];
|
||||
if (!pattern) {
|
||||
console.info(`Pattern #${match.patternId} does not exist`);
|
||||
return;
|
||||
}
|
||||
const firstTag = `<span style="background-color: ${pattern.color}">`;
|
||||
const secondTag = '</span>';
|
||||
|
||||
const positionStart = match.startPosition + offset;
|
||||
raw = raw.substring(0, positionStart) + firstTag + raw.substring(positionStart);
|
||||
offset += firstTag.length;
|
||||
|
||||
const positionEnd = match.endPosition + offset + 1;
|
||||
raw = raw.substring(0, positionEnd) + secondTag + raw.substring(positionEnd);
|
||||
offset += secondTag.length;
|
||||
});
|
||||
return raw;
|
||||
},
|
||||
|
||||
copyPythonBytes() {
|
||||
const data = 'b\'' + atob(this.packet.content)
|
||||
.split('')
|
||||
.map((aChar) => {
|
||||
return '\\x' + ('0' + aChar.charCodeAt(0).toString(16)).slice(-2);
|
||||
})
|
||||
.join('') + '\'';
|
||||
this.copyContent(data);
|
||||
},
|
||||
copyRaw() {
|
||||
this.copyContent(Buffer.from(this.packet.content, 'base64').toString('hex'));
|
||||
},
|
||||
copyText() {
|
||||
this.copyContent(atob(this.packet.content));
|
||||
},
|
||||
copyContent(data) {
|
||||
const tempEl = document.createElement('textarea');
|
||||
tempEl.value = data; // Chrome
|
||||
tempEl.textContent = data; // Firefox
|
||||
document.body.appendChild(tempEl);
|
||||
tempEl.select();
|
||||
const result = document.execCommand('copy');
|
||||
document.body.removeChild(tempEl);
|
||||
console.debug('Copy result is', result);
|
||||
},
|
||||
},
|
||||
};
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.packet-addresses {
|
||||
font-family: "Ubuntu Mono", "Lucida Console", monospace;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.packet-outgoing {
|
||||
background: var(--response-packet-color);
|
||||
box-shadow: 0 0 5px 5px var(--response-packet-color);
|
||||
}
|
||||
|
||||
.packet-incoming {
|
||||
background: var(--request-packet-color);
|
||||
box-shadow: 0 0 5px 5px var(--request-packet-color);
|
||||
}
|
||||
|
||||
p {
|
||||
font-family: "Ubuntu Mono", "Lucida Console", monospace;
|
||||
font-size: 100%;
|
||||
margin-bottom: 10px;
|
||||
padding-bottom: 5px;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
button {
|
||||
padding: 0;
|
||||
top: -0.1em;
|
||||
position: relative;
|
||||
margin-left: 5px;
|
||||
}
|
||||
</style>
|
||||
208
frontend/src/components/PatternsDropdown.vue
Normal file
@ -0,0 +1,208 @@
|
||||
<template>
|
||||
<b-dropdown no-flip text="Patterns" block variant="dark" class="col-sm-1 mr-0 p-0">
|
||||
<li role="presentation" style="padding-left: 0.5em; padding-right: 0.5em;">
|
||||
<button role="menuitem" type="button" class="btn btn-sm btn-primary btn-block"
|
||||
@click.stop.prevent="showAddPattern">
|
||||
<i class="fas fa-plus"/>
|
||||
</button>
|
||||
</li>
|
||||
<b-dropdown-divider/>
|
||||
<b-dropdown-item-button @click.stop.prevent="resetPatternSelection">
|
||||
<strong>All streams</strong>
|
||||
</b-dropdown-item-button>
|
||||
<b-dropdown-item-button v-for="pattern in existingPatterns"
|
||||
:key="pattern.id" @click.stop.prevent="openPattern(pattern)"
|
||||
:class="{ 'ignore-pattern' : pattern.actionType === 'IGNORE' }">
|
||||
|
||||
<strong v-if="pattern.enabled" :style="getPatternColor">{{ pattern.name }}</strong>
|
||||
<s v-else :style="`color: ${pattern.color};`">{{ pattern.name }}</s>:
|
||||
|
||||
<code>{{ getSearchTypeValue(pattern.searchType, pattern.value) }}</code>;
|
||||
<span v-if="pattern.searchTarget && pattern.searchTarget !== 'CONTENT'">{{ getSearchTargetLabel(pattern.searchTarget) }}; </span>
|
||||
|
||||
<template v-if="pattern.actionType === 'FIND'">search </template>
|
||||
<template v-else>ignore </template>
|
||||
|
||||
<template v-if="pattern.directionType === 'BOTH'">anywhere </template>
|
||||
<template v-else-if="pattern.directionType === 'INPUT'">in request </template>
|
||||
<template v-else>in response </template>
|
||||
|
||||
<template v-if="pattern.serviceId === null">of any service</template>
|
||||
<template v-else>of service {{ getServiceName(pattern.serviceId) }} #{{ pattern.serviceId }}</template>
|
||||
|
||||
<div class="float-right" style="margin-left: 2em;">
|
||||
<button type="button" class="btn btn-outline-info btn-sm mr-1"
|
||||
@click.stop.prevent="showEditPattern(pattern)"
|
||||
title="Edit pattern">
|
||||
<i class="fas fa-edit"></i>
|
||||
</button>
|
||||
|
||||
<button v-if="pattern.actionType === 'FIND'" type="button" class="btn btn-outline-warning btn-sm mr-1"
|
||||
@click.stop.prevent="showLookBack(pattern)"
|
||||
title="Apply pattern to older streams">
|
||||
<i class="fas fa-backward"></i>
|
||||
</button>
|
||||
|
||||
<button v-if="pattern.enabled" type="button" class="btn btn-outline-danger btn-sm mr-1"
|
||||
@click.stop.prevent="togglePattern(pattern)"
|
||||
title="Stop matching streams with this pattern">
|
||||
<i class="fas fa-pause"></i>
|
||||
</button>
|
||||
<button v-else type="button" class="btn btn-outline-success btn-sm mr-1"
|
||||
@click.stop.prevent="togglePattern(pattern)"
|
||||
title="Start matching streams with this pattern again">
|
||||
<i class="fas fa-play"></i>
|
||||
</button>
|
||||
|
||||
<button type="button" class="btn btn-outline-danger btn-sm"
|
||||
@click.stop.prevent="deletePattern(pattern)"
|
||||
title="Permanently delete this pattern">
|
||||
<i class="fas fa-trash"></i>
|
||||
</button>
|
||||
</div>
|
||||
</b-dropdown-item-button>
|
||||
|
||||
<PatternModal :creating="patternModalIsCreating" :initial-pattern="patternModalEditingPattern" />
|
||||
<LookBack :pattern-id="patternIdForLookback" />
|
||||
</b-dropdown>
|
||||
</template>
|
||||
<!--suppress JSUnresolvedReference -->
|
||||
<script>
|
||||
import PatternModal from '@/views/PatternModal.vue';
|
||||
import LookBack from '@/views/LookBack.vue';
|
||||
|
||||
export default {
|
||||
name: 'PatternsDropdown',
|
||||
components: {LookBack, PatternModal, },
|
||||
data() {
|
||||
return {
|
||||
patternModalIsCreating: true,
|
||||
patternModalEditingPattern: {},
|
||||
patternIdForLookback: 0,
|
||||
};
|
||||
},
|
||||
mounted() {
|
||||
this.updatePatterns();
|
||||
},
|
||||
computed: {
|
||||
existingPatterns: function () {
|
||||
return this.$store.state.patterns.filter(p => !p.deleted)
|
||||
},
|
||||
},
|
||||
methods: {
|
||||
getSearchTargetLabel(target) {
|
||||
return {SOURCE_IP: 'source IP', DESTINATION_IP: 'destination IP', EITHER_IP: 'either IP address',}[target];
|
||||
},
|
||||
getSearchTypeValue(searchType, value) {
|
||||
if (searchType === 'REGEX') return `/${value}/`;
|
||||
else if (searchType === 'SUBSTRING') return `'${value}'`;
|
||||
else return `0x${value}`;
|
||||
},
|
||||
getPatternColor(pattern) {
|
||||
if (pattern.actionType === 'FIND') {
|
||||
return `color: ${pattern.color};`;
|
||||
} else {
|
||||
return `color: inherit;`;
|
||||
}
|
||||
},
|
||||
getServiceName(port) {
|
||||
return this.$store.state.services.find(o => o.port === port)?.name ?? '<Deleted service>'
|
||||
},
|
||||
showAddPattern() {
|
||||
this.patternModalIsCreating = true;
|
||||
this.patternModalEditingPattern = {};
|
||||
this.$bvModal.show('patternModal');
|
||||
console.debug('Showing patternModal (create)');
|
||||
},
|
||||
showEditPattern(pattern) {
|
||||
this.patternModalIsCreating = false;
|
||||
this.patternModalEditingPattern = {};
|
||||
this.$nextTick(() => {
|
||||
this.patternModalEditingPattern = pattern;
|
||||
this.$bvModal.show('patternModal');
|
||||
console.debug('Showing patternModal (edit)');
|
||||
});
|
||||
},
|
||||
showLookBack(pattern) {
|
||||
this.patternIdForLookback = pattern.id;
|
||||
this.$bvModal.show('lookBackModal');
|
||||
},
|
||||
updatePatterns() {
|
||||
this.$http.get('pattern/')
|
||||
.then(r => this.$store.commit('setPatterns', r.data))
|
||||
.catch(e => {
|
||||
this.$bvToast.toast(`Failed to load patterns: ${e}`, {
|
||||
title: 'Error',
|
||||
variant: 'danger',
|
||||
});
|
||||
console.error('Failed to load patterns:', e);
|
||||
});
|
||||
},
|
||||
openPattern(pattern) {
|
||||
if (pattern.actionType === 'IGNORE') {
|
||||
return;
|
||||
}
|
||||
|
||||
console.debug('Opening pattern w/ id', pattern.id);
|
||||
this.$router.push({
|
||||
name: 'stream',
|
||||
params: this.$route.params,
|
||||
query: {pattern: pattern.id,},
|
||||
}, () => {});
|
||||
},
|
||||
resetPatternSelection() {
|
||||
console.debug('Resetting pattern selection');
|
||||
this.$router.push({
|
||||
name: 'stream',
|
||||
params: this.$route.params,
|
||||
}, () => {});
|
||||
},
|
||||
togglePattern(pattern) {
|
||||
const enabled = !pattern.enabled;
|
||||
console.debug('Toggling pattern', pattern);
|
||||
this.$http.post(`pattern/${pattern.id}/enable`, null, {
|
||||
params: {
|
||||
enabled,
|
||||
},
|
||||
})
|
||||
.then(response => {
|
||||
const data = response.data;
|
||||
console.debug('Done toggling pattern', data);
|
||||
this.$emit('patternAddComplete');
|
||||
}).catch(e => {
|
||||
this.$bvToast.toast(`Failed to toggle pattern: ${e}`, {
|
||||
title: 'Error',
|
||||
variant: 'danger',
|
||||
});
|
||||
console.error('Failed to toggle pattern', e);
|
||||
});
|
||||
},
|
||||
deletePattern(pattern) {
|
||||
console.debug('Deleting pattern', pattern);
|
||||
|
||||
this.$http.delete(`pattern/${pattern.id}`, null)
|
||||
.then(() => {
|
||||
console.debug('Done deleting pattern');
|
||||
this.$emit('patternAddComplete');
|
||||
}).catch(e => {
|
||||
this.$bvToast.toast(`Failed to delete pattern: ${e}`, {
|
||||
title: 'Error',
|
||||
variant: 'danger',
|
||||
});
|
||||
console.error('Failed to delete pattern', e);
|
||||
});
|
||||
},
|
||||
},
|
||||
};
|
||||
</script>
|
||||
<style scoped>
|
||||
code {
|
||||
font-family: "Ubuntu Mono", "Lucida Console", monospace;
|
||||
font-size: 100%;
|
||||
}
|
||||
</style>
|
||||
<style>
|
||||
.ignore-pattern button {
|
||||
cursor: default !important;
|
||||
}
|
||||
</style>
|
||||
355
frontend/src/components/Sidebar.vue
Normal file
@ -0,0 +1,355 @@
|
||||
<template>
|
||||
<nav class="col-sm-3 d-none d-sm-block sidebar"
|
||||
:class="this.$store.state.theme === 'light' ? 'bg-light' : 'bg-dark'">
|
||||
<div class="m-2">
|
||||
<button type="button" class="btn btn-sm"
|
||||
:title="this.$store.state.pause ? 'Continue' : 'Pause new streams'"
|
||||
:class="this.$store.state.pause ? 'btn-danger' : 'btn-outline-success'"
|
||||
@click.stop.prevent="togglePause">
|
||||
<i :class="this.$store.state.pause ? 'fas fa-play' : 'fas fa-pause'"/>
|
||||
</button>
|
||||
<button type="button" class="btn btn-sm ml-1"
|
||||
:title="this.$store.state.displayFavoritesOnly ? 'Show all streams' : 'Show only favorite streams'"
|
||||
:class="this.$store.state.displayFavoritesOnly ? 'btn-danger' : 'btn-outline-danger'"
|
||||
@click.stop.prevent="toggleFavorites">
|
||||
<i class="fas fa-star"/>
|
||||
</button>
|
||||
<button type="button" class="btn btn-outline-primary btn-sm ml-1"
|
||||
:title="this.$store.state.hexdumpMode ? 'Switch to text view' : 'Switch to hexdump view'"
|
||||
@click.stop.prevent="toggleHexdump">
|
||||
<i :class="this.$store.state.hexdumpMode ? 'far fa-file-code' : 'fas fa-align-left'"/>
|
||||
</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-warning ml-1" v-if="!this.$store.state.pcapStarted"
|
||||
title="Start pcap file processing"
|
||||
@click.stop.prevent="startPcap">
|
||||
<i class="fas fa-arrow-circle-down"/>
|
||||
</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-info" style="float: right;"
|
||||
title="Scroll to top"
|
||||
@click.stop.prevent="scrollUp">
|
||||
<i class="fas fa-angle-double-up"/>
|
||||
</button>
|
||||
</div>
|
||||
<div class="sidebar-sticky">
|
||||
<ul class="nav flex-column">
|
||||
<SidebarStream v-for="stream in streams"
|
||||
:key="stream.id"
|
||||
:stream="stream"/>
|
||||
<infinite-loading @infinite="infiniteLoadingHandler" ref="infiniteLoader"/>
|
||||
</ul>
|
||||
</div>
|
||||
</nav>
|
||||
</template>
|
||||
|
||||
<script>
|
||||
import SidebarStream from './SidebarStream';
|
||||
|
||||
export default {
|
||||
name: 'Sidebar',
|
||||
props: ['servicePort', 'streamId',],
|
||||
data() {
|
||||
return {
|
||||
streams: [],
|
||||
requestGeneration: 0,
|
||||
reconcileTimeout: null,
|
||||
reconcileInterval: null,
|
||||
reconcileInFlight: false,
|
||||
reconcileAgain: false,
|
||||
visibilityCallback: null,
|
||||
navigationKeysCallback: (e) => {
|
||||
if (!e.ctrlKey) return;
|
||||
if (e.key === 'ArrowUp') {
|
||||
e.preventDefault();
|
||||
const index = this.streams.findIndex(e => e.id === this.$route.params.streamId);
|
||||
const newStream = this.streams[index - 1];
|
||||
if (!newStream) return;
|
||||
const newId = newStream.id;
|
||||
this.$router.push({
|
||||
name: 'stream',
|
||||
params: {servicePort: this.$route.params.servicePort, streamId: newId,},
|
||||
query: this.$route.query,
|
||||
});
|
||||
document.getElementById(`stream-${newId}`).scrollIntoView({
|
||||
behavior: 'smooth',
|
||||
block: 'center',
|
||||
inline: 'center',
|
||||
});
|
||||
} else if (e.key === 'ArrowDown') {
|
||||
e.preventDefault();
|
||||
const index = this.streams.findIndex(e => e.id === this.$route.params.streamId);
|
||||
const newStream = this.streams[index + 1];
|
||||
if (!newStream) return;
|
||||
const newId = newStream.id;
|
||||
this.$router.push({
|
||||
name: 'stream',
|
||||
params: {servicePort: this.$route.params.servicePort, streamId: newId,},
|
||||
query: this.$route.query,
|
||||
});
|
||||
document.getElementById(`stream-${newId}`).scrollIntoView({
|
||||
behavior: 'smooth',
|
||||
block: 'center',
|
||||
inline: 'center',
|
||||
});
|
||||
} else if (e.key === 'Home') {
|
||||
const newStream = this.streams[0];
|
||||
if (!newStream) return;
|
||||
const newId = newStream.id;
|
||||
this.$router.push({
|
||||
name: 'stream',
|
||||
params: {servicePort: this.$route.params.servicePort, streamId: newId,},
|
||||
query: this.$route.query,
|
||||
});
|
||||
document.getElementById(`stream-${newId}`).scrollIntoView({
|
||||
behavior: 'smooth',
|
||||
block: 'center',
|
||||
inline: 'center',
|
||||
});
|
||||
} else if (e.key === 'End') {
|
||||
const newStream = this.streams[this.streams.length - 1];
|
||||
if (!newStream) return;
|
||||
const newId = newStream.id;
|
||||
this.$router.push({
|
||||
name: 'stream',
|
||||
params: {servicePort: this.$route.params.servicePort, streamId: newId,},
|
||||
query: this.$route.query,
|
||||
});
|
||||
document.getElementById(`stream-${newId}`).scrollIntoView({
|
||||
behavior: 'smooth',
|
||||
block: 'center',
|
||||
inline: 'center',
|
||||
});
|
||||
}
|
||||
},
|
||||
};
|
||||
},
|
||||
watch: {
|
||||
'$route.params.servicePort': function () {
|
||||
console.debug('Port reselected');
|
||||
this.requestGeneration++;
|
||||
this.streams = [];
|
||||
this.$refs.infiniteLoader.stateChanger.reset();
|
||||
},
|
||||
'$route.query.pattern': function () {
|
||||
console.debug('Pattern selected');
|
||||
this.requestGeneration++;
|
||||
this.streams = [];
|
||||
this.$refs.infiniteLoader.stateChanger.reset();
|
||||
},
|
||||
},
|
||||
methods: {
|
||||
infiniteLoadingHandler($state) {
|
||||
const generation = this.requestGeneration;
|
||||
const ourStreams = this.streams;
|
||||
const pageSize = this.$store.state.pageSize;
|
||||
|
||||
let startsFrom;
|
||||
if (ourStreams?.length && ourStreams[ourStreams.length - 1]) {
|
||||
startsFrom = ourStreams[ourStreams.length - 1].id;
|
||||
} else {
|
||||
startsFrom = null;
|
||||
}
|
||||
|
||||
this.$http.post(`/stream/${this.$route?.params?.servicePort || 'all'}`, {
|
||||
startingFrom: startsFrom,
|
||||
pageSize: pageSize,
|
||||
pattern: this.$route.query.pattern ? {id: this.$route.query.pattern,} : null,
|
||||
favorites: this.$store.state.displayFavoritesOnly,
|
||||
}).then(r => {
|
||||
if (generation !== this.requestGeneration) return;
|
||||
const data = r.data;
|
||||
if (data?.length === 0) {
|
||||
console.log('Finished loading streams (empty page)');
|
||||
return $state.complete();
|
||||
}
|
||||
|
||||
if (data[0]?.id === this.streams[0]?.id) {
|
||||
console.log('Finished loading streams (overlap detected)');
|
||||
return $state.complete();
|
||||
}
|
||||
|
||||
const loadedIds = new Set(this.streams.map(stream => stream.id));
|
||||
this.streams.push(...data.filter(stream => !loadedIds.has(stream.id)));
|
||||
|
||||
if (data.length < pageSize) {
|
||||
// this was the last page
|
||||
console.log('Finished loading streams (last page was not full)');
|
||||
$state.complete();
|
||||
} else {
|
||||
console.log('Loaded another page of streams');
|
||||
$state.loaded()
|
||||
}
|
||||
}).catch(e => {
|
||||
if (generation !== this.requestGeneration) return;
|
||||
this.$bvToast.toast(`Failed to load portion of streams: ${e}`, {
|
||||
title: 'Error',
|
||||
variant: 'danger',
|
||||
});
|
||||
console.error('Failed to load portion of streams:', e);
|
||||
return $state.error();
|
||||
});
|
||||
},
|
||||
toggleFavorites() {
|
||||
this.$store.commit('toggleDisplayFavoritesOnly');
|
||||
this.requestGeneration++;
|
||||
this.streams = [];
|
||||
this.$refs.infiniteLoader.stateChanger.reset();
|
||||
},
|
||||
togglePause() {
|
||||
this.$store.commit('togglePause');
|
||||
},
|
||||
toggleHexdump() {
|
||||
this.$store.commit('toggleHexdumpMode');
|
||||
},
|
||||
startPcap() {
|
||||
this.$http.post('pcap/start')
|
||||
.then(() => {
|
||||
this.$store.commit('startPcap');
|
||||
}).catch(e => {
|
||||
this.$bvToast.toast(`Failed to start pcap: ${e}`, {
|
||||
title: 'Error',
|
||||
variant: 'danger',
|
||||
});
|
||||
console.error('Failed to start pcap', e);
|
||||
});
|
||||
},
|
||||
addStreamFromWs(stream) {
|
||||
const currentPort = parseInt(this.$route?.params?.servicePort, 10);
|
||||
if (currentPort && currentPort !== stream.service) return;
|
||||
if (this.$store.state.displayFavoritesOnly || this.$store.state.pause) return;
|
||||
if (this.$route.query.pattern && !(stream.foundPatternsIds || []).includes(Number(this.$route.query.pattern))) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!this.streams.some(existing => existing.id === stream.id)) {
|
||||
this.streams.unshift(stream);
|
||||
}
|
||||
},
|
||||
scheduleReconciliation(delay = 250) {
|
||||
if (this.reconcileTimeout !== null) return;
|
||||
this.reconcileTimeout = setTimeout(() => {
|
||||
this.reconcileTimeout = null;
|
||||
this.reconcileStreams();
|
||||
}, delay);
|
||||
},
|
||||
async reconcileStreams() {
|
||||
if (this.reconcileInFlight) {
|
||||
this.reconcileAgain = true;
|
||||
return;
|
||||
}
|
||||
|
||||
const snapshotIds = new Set(this.streams.map(stream => Number(stream.id)));
|
||||
const selectedId = Number(this.$route.params.streamId);
|
||||
if (Number.isSafeInteger(selectedId) && selectedId > 0) snapshotIds.add(selectedId);
|
||||
if (snapshotIds.size === 0) return;
|
||||
|
||||
this.reconcileInFlight = true;
|
||||
const generation = this.requestGeneration;
|
||||
try {
|
||||
const ids = Array.from(snapshotIds);
|
||||
const availableIds = new Set();
|
||||
for (let offset = 0; offset < ids.length; offset += 500) {
|
||||
const response = await this.$http.post('/stream/available', {
|
||||
ids: ids.slice(offset, offset + 500),
|
||||
});
|
||||
response.data.forEach(id => availableIds.add(Number(id)));
|
||||
}
|
||||
|
||||
if (generation !== this.requestGeneration) return;
|
||||
this.streams = this.streams.filter(stream =>
|
||||
!snapshotIds.has(Number(stream.id)) || availableIds.has(Number(stream.id)));
|
||||
|
||||
if (snapshotIds.has(selectedId) && !availableIds.has(selectedId)
|
||||
&& Number(this.$route.params.streamId) === selectedId) {
|
||||
this.requestGeneration++;
|
||||
this.$bvToast.toast('The selected stream was removed by retention cleanup', {
|
||||
title: 'Stream removed',
|
||||
variant: 'info',
|
||||
autoHideDelay: 5000,
|
||||
});
|
||||
this.$router.replace({
|
||||
name: 'stream',
|
||||
params: {servicePort: this.$route.params.servicePort,},
|
||||
query: this.$route.query,
|
||||
}).catch(() => {});
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('Failed to reconcile retained streams:', e);
|
||||
} finally {
|
||||
this.reconcileInFlight = false;
|
||||
if (this.reconcileAgain) {
|
||||
this.reconcileAgain = false;
|
||||
this.scheduleReconciliation(0);
|
||||
}
|
||||
}
|
||||
},
|
||||
scrollUp() {
|
||||
const newStream = this.streams[0];
|
||||
if (!newStream) return;
|
||||
const newId = newStream.id;
|
||||
this.$router.push({
|
||||
name: 'stream',
|
||||
params: {servicePort: this.$route.params.servicePort, streamId: newId,},
|
||||
query: this.$route.query,
|
||||
});
|
||||
document.getElementById(`stream-${newId}`).scrollIntoView({
|
||||
behavior: 'smooth',
|
||||
block: 'center',
|
||||
inline: 'center',
|
||||
});
|
||||
},
|
||||
},
|
||||
mounted() {
|
||||
document.addEventListener('keydown', this.navigationKeysCallback);
|
||||
this.visibilityCallback = () => {
|
||||
if (document.visibilityState === 'visible') this.scheduleReconciliation(0);
|
||||
};
|
||||
document.addEventListener('visibilitychange', this.visibilityCallback);
|
||||
this.reconcileInterval = setInterval(() => this.scheduleReconciliation(0), 60_000);
|
||||
|
||||
this.$http.get('pcap/started')
|
||||
.then(r => this.$store.state.pcapStarted = r.data)
|
||||
.catch(e => {
|
||||
console.error('Failed to get pcap status, defaulting to true:', e);
|
||||
this.$store.state.pcapStarted = true;
|
||||
});
|
||||
},
|
||||
beforeDestroy() {
|
||||
document.removeEventListener('keydown', this.navigationKeysCallback);
|
||||
document.removeEventListener('visibilitychange', this.visibilityCallback);
|
||||
if (this.reconcileTimeout !== null) clearTimeout(this.reconcileTimeout);
|
||||
if (this.reconcileInterval !== null) clearInterval(this.reconcileInterval);
|
||||
this.requestGeneration++;
|
||||
},
|
||||
components: {
|
||||
SidebarStream,
|
||||
},
|
||||
};
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.sidebar {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
bottom: 0;
|
||||
left: 0;
|
||||
z-index: 100; /* Behind the navbar */
|
||||
padding: 40px 0 0; /* Height of navbar */
|
||||
box-shadow: inset -1px 0 0 rgba(0, 0, 0, .1);
|
||||
}
|
||||
|
||||
.sidebar-sticky {
|
||||
position: relative;
|
||||
top: 0;
|
||||
height: calc(100vh - 85px);
|
||||
/*padding-top: .5rem;*/
|
||||
overflow-x: hidden;
|
||||
overflow-y: auto; /* Scrollable contents if viewport is shorter than content. */
|
||||
}
|
||||
|
||||
@supports (position: sticky) {
|
||||
.sidebar-sticky {
|
||||
position: sticky;
|
||||
}
|
||||
}
|
||||
|
||||
</style>
|
||||
152
frontend/src/components/SidebarStream.vue
Normal file
@ -0,0 +1,152 @@
|
||||
<template>
|
||||
<li :id="`stream-${stream.id}`" class="nav-item stream-item" :class="{highlight: favorite0}">
|
||||
<router-link class="nav-link"
|
||||
:to="{name: 'stream', params: {servicePort: this.stream.service, streamId: this.stream.id}, query: this.$route.query}">
|
||||
<a @click.stop.prevent="toggleFavorite">
|
||||
<i class="fa-star" style="color: #DC3545" :class="favorite0 ? 'fas' : 'far'"/>
|
||||
</a>
|
||||
{{ stream.id }} {{ stream.protocol }}
|
||||
<template v-if="stream.ttl">TTL {{ stream.ttl }}</template>
|
||||
<template v-if="shouldShowServiceName"><br/>{{ getServiceName(stream.service) }} #{{stream.service}}</template>
|
||||
<br/>
|
||||
{{ dateToText(stream.startTimestamp) }}
|
||||
<template v-if="!dateMatches(stream.startTimestamp, stream.endTimestamp)">
|
||||
- {{ dateToText(stream.endTimestamp, dayMatches(stream.startTimestamp, stream.endTimestamp)) }}
|
||||
</template>
|
||||
<template v-if="stream.userAgentHash"><br/>UA: {{ stream.userAgentHash }}</template>
|
||||
<br/>
|
||||
{{ stream.sizeBytes }} bytes in {{ stream.packetsCount }} packets
|
||||
<br/>
|
||||
<span v-for="pattern in notDeletedFoundPatterns"
|
||||
:key="pattern.id"
|
||||
:style="`color: ${pattern.color};`">
|
||||
{{ pattern.name }}
|
||||
</span>
|
||||
</router-link>
|
||||
</li>
|
||||
</template>
|
||||
|
||||
<script>
|
||||
export default {
|
||||
name: 'SidebarStream',
|
||||
props: {
|
||||
stream: {
|
||||
id: Number(),
|
||||
protocol: String(),
|
||||
service: Number(),
|
||||
startTimestamp: Number(),
|
||||
endTimestamp: Number(),
|
||||
foundPatternsIds: Array(),
|
||||
favorite: Boolean(),
|
||||
ttl: Number(),
|
||||
userAgentHash: String(),
|
||||
sizeBytes: Number(),
|
||||
packetsCount: Number(),
|
||||
},
|
||||
},
|
||||
computed: {
|
||||
shouldShowServiceName: function () {
|
||||
return this.$route?.params?.servicePort === undefined;
|
||||
},
|
||||
foundPatterns: function () {
|
||||
return this.$store.state.patterns.filter(p => this.stream.foundPatternsIds.includes(p.id));
|
||||
},
|
||||
notDeletedFoundPatterns: function () {
|
||||
return this.foundPatterns.filter(p => !p.deleted);
|
||||
},
|
||||
},
|
||||
data: function () {
|
||||
return {
|
||||
favorite0: this.stream.favorite,
|
||||
getServiceName: function (port) {
|
||||
return this.$store.state.services.find(o => o.port === port)?.name ?? '<Deleted service>'
|
||||
},
|
||||
};
|
||||
},
|
||||
methods: {
|
||||
dateToText(unixTimestamp, short = false) {
|
||||
const date = new Date(unixTimestamp);
|
||||
const options = {
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
second: '2-digit',
|
||||
};
|
||||
if (!short && !this.isToday(date)) {
|
||||
options.month = '2-digit';
|
||||
options.day = '2-digit';
|
||||
return date.toLocaleDateString('ru-RU', options);
|
||||
}
|
||||
return date.toLocaleTimeString('ru-RU', options);
|
||||
},
|
||||
|
||||
isToday(someDate) {
|
||||
const today = new Date();
|
||||
return someDate.getDate() === today.getDate() &&
|
||||
someDate.getMonth() === today.getMonth() &&
|
||||
someDate.getFullYear() === today.getFullYear();
|
||||
},
|
||||
|
||||
dayMatches(rFirst, rSecond) {
|
||||
const first = new Date(rFirst);
|
||||
const second = new Date(rSecond);
|
||||
|
||||
return first.getDate() === second.getDate() &&
|
||||
first.getMonth() === second.getMonth() &&
|
||||
first.getFullYear() === second.getFullYear();
|
||||
},
|
||||
|
||||
dateMatches(rFirst, rSecond) {
|
||||
const first = new Date(rFirst);
|
||||
const second = new Date(rSecond);
|
||||
return first.getDate() === second.getDate() &&
|
||||
first.getMonth() === second.getMonth() &&
|
||||
first.getFullYear() === second.getFullYear() &&
|
||||
first.getHours() === second.getHours() &&
|
||||
first.getMinutes() === second.getMinutes() &&
|
||||
first.getSeconds() === second.getSeconds();
|
||||
},
|
||||
|
||||
toggleFavorite() {
|
||||
this.$http.post(`stream/${this.stream.id}/${this.favorite0 ? 'unfavorite' : 'favorite'}`)
|
||||
.then(() => this.favorite0 = !this.favorite0)
|
||||
.catch(e => {
|
||||
this.$bvToast.toast(`Failed to fav service: ${e}`, {
|
||||
title: 'Error',
|
||||
variant: 'danger',
|
||||
});
|
||||
console.error('Failed to fav service', e);
|
||||
});
|
||||
},
|
||||
},
|
||||
};
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.nav-link {
|
||||
font-weight: 500;
|
||||
color: #333;
|
||||
}
|
||||
|
||||
[data-theme=dark] .nav-link {
|
||||
color: #f8f9fa;
|
||||
}
|
||||
|
||||
/*noinspection CssUnusedSymbol*/
|
||||
.nav-link.active {
|
||||
color: #007BFF;
|
||||
}
|
||||
|
||||
.highlight {
|
||||
background: rgba(220, 53, 69, 0.3);
|
||||
box-shadow: 0 0 1px 1px rgba(255, 119, 182, 0.3);
|
||||
}
|
||||
|
||||
.stream-item {
|
||||
transition: all .5s;
|
||||
box-shadow: 0 24px 3px -25px #4b4e51;
|
||||
}
|
||||
|
||||
[data-theme=dark] .stream-item {
|
||||
box-shadow: 0 24px 3px -25px #84898E;
|
||||
}
|
||||
</style>
|
||||
127
frontend/src/components/ThemeButton.vue
Normal file
@ -0,0 +1,127 @@
|
||||
<template>
|
||||
<div>
|
||||
<input
|
||||
@change="toggleTheme"
|
||||
id="checkbox"
|
||||
type="checkbox"
|
||||
class="switch-checkbox"
|
||||
/>
|
||||
<label for="checkbox" class="switch-label">
|
||||
<span class="checkbox-emoji">🌙</span>
|
||||
<span class="checkbox-emoji">☀️</span>
|
||||
<div
|
||||
class="switch-toggle"
|
||||
:class="{ 'switch-toggle-checked': chosenTheme === 'dark' }"
|
||||
></div>
|
||||
</label>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script>
|
||||
|
||||
export default {
|
||||
mounted() {
|
||||
this.initTheme();
|
||||
},
|
||||
|
||||
computed: {
|
||||
chosenTheme: {
|
||||
get() {
|
||||
return this.$store.state.theme || this.detectTheme();
|
||||
},
|
||||
|
||||
set(theme) {
|
||||
this.$store.commit('setTheme', theme);
|
||||
this.displayTheme(theme);
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
methods: {
|
||||
toggleTheme() {
|
||||
const activeTheme = this.chosenTheme;
|
||||
|
||||
if (activeTheme === "light") {
|
||||
this.chosenTheme = 'dark';
|
||||
} else {
|
||||
this.chosenTheme = 'light';
|
||||
}
|
||||
|
||||
console.debug('Toggling theme from ', activeTheme, ' to ', this.chosenTheme);
|
||||
},
|
||||
|
||||
initTheme() {
|
||||
this.displayTheme(this.chosenTheme);
|
||||
},
|
||||
|
||||
detectTheme() {
|
||||
return window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light';
|
||||
},
|
||||
|
||||
displayTheme(theme) {
|
||||
document.body.setAttribute('data-theme', theme);
|
||||
},
|
||||
},
|
||||
};
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.switch-checkbox {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.switch-label {
|
||||
align-items: center;
|
||||
background: var(--button-text-primary-color);
|
||||
border: calc(var(--button-element-size) * 0.025) solid var(--button-accent-color);
|
||||
border-radius: var(--button-element-size);
|
||||
cursor: pointer;
|
||||
display: flex;
|
||||
font-size: calc(var(--button-element-size) * 0.3);
|
||||
height: calc(var(--button-element-size) * 0.35);
|
||||
position: relative;
|
||||
padding: calc(var(--button-element-size) * 0.1);
|
||||
transition: background 0.5s ease;
|
||||
justify-content: space-between;
|
||||
width: var(--button-element-size);
|
||||
z-index: 1;
|
||||
box-sizing: content-box;
|
||||
}
|
||||
|
||||
.switch-toggle {
|
||||
position: absolute;
|
||||
background-color: var(--button-background-color-primary);
|
||||
border-radius: 50%;
|
||||
top: calc(var(--button-element-size) * 0.07);
|
||||
left: calc(var(--button-element-size) * 0.07);
|
||||
height: calc(var(--button-element-size) * 0.4);
|
||||
width: calc(var(--button-element-size) * 0.4);
|
||||
transform: translateX(0);
|
||||
transition: transform 0.3s ease, background-color 0.5s ease;
|
||||
}
|
||||
|
||||
.switch-toggle-checked {
|
||||
transform: translateX(calc(var(--button-element-size) * 0.6)) !important;
|
||||
}
|
||||
|
||||
.checkbox-emoji {
|
||||
color: transparent;
|
||||
text-shadow: 0 0 0 var(--button-background-color-primary);
|
||||
}
|
||||
|
||||
>>> {
|
||||
--button-element-size: 3rem;
|
||||
|
||||
--button-background-color-primary: #ebebeb;
|
||||
--button-background-color-secondary: #fafafa;
|
||||
--button-accent-color: #cacaca;
|
||||
--button-text-primary-color: #222;
|
||||
}
|
||||
|
||||
[data-theme=dark] * {
|
||||
--button-background-color-primary: #1e1e1e;
|
||||
--button-background-color-secondary: #2d2d30;
|
||||
--button-accent-color: #3f3f3f;
|
||||
--button-text-primary-color: #ddd;
|
||||
}
|
||||
</style>
|
||||
47
frontend/src/main.js
Normal file
@ -0,0 +1,47 @@
|
||||
import Vue from 'vue';
|
||||
import BootstrapVue from 'bootstrap-vue';
|
||||
import Axios from 'axios';
|
||||
import InfiniteLoading from 'vue-infinite-loading';
|
||||
import App from './App.vue';
|
||||
import router from './router';
|
||||
|
||||
import store from './store';
|
||||
import 'bootstrap/dist/css/bootstrap.css';
|
||||
import 'bootstrap-vue/dist/bootstrap-vue.css';
|
||||
import '@fortawesome/fontawesome-free/css/all.css';
|
||||
import "bootstrap-darkmode/scss/darktheme.scss";
|
||||
|
||||
Vue.config.productionTip = false;
|
||||
Vue.use(BootstrapVue);
|
||||
Vue.use(InfiniteLoading, {
|
||||
slots: {
|
||||
noResults: 'No results...',
|
||||
noMore: 'No more data',
|
||||
error: 'An error has occurred!',
|
||||
errorBtnText: 'Retry',
|
||||
},
|
||||
props: {
|
||||
spinner: 'waveDots',
|
||||
},
|
||||
});
|
||||
|
||||
const axiosInstance = Axios.create({
|
||||
baseURL: '/api', // TO!DO: edit for release!
|
||||
// baseURL: 'http://192.168.79.131:65000/api',
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
timeout: 5000,
|
||||
withCredentials: true,
|
||||
maxContentLength: 20000,
|
||||
});
|
||||
// noinspection JSUnusedGlobalSymbols
|
||||
Vue.prototype.$http = axiosInstance;
|
||||
|
||||
// noinspection JSUnusedGlobalSymbols
|
||||
new Vue({
|
||||
router,
|
||||
store,
|
||||
render: h => h(App),
|
||||
}).$mount('#app');
|
||||
118
frontend/src/objectAssignDeep.js
Normal file
@ -0,0 +1,118 @@
|
||||
// https://github.com/saikojosh/Object-Assign-Deep/blob/master/objectAssignDeep.js
|
||||
|
||||
/*
|
||||
* A unified way of returning a string that describes the type of the given variable.
|
||||
*/
|
||||
function getTypeOf(input) {
|
||||
|
||||
if (input === null) {
|
||||
return 'null';
|
||||
} else if (typeof input === 'undefined') {
|
||||
return 'undefined';
|
||||
} else if (typeof input === 'object') {
|
||||
return (Array.isArray(input) ? 'array' : 'object');
|
||||
}
|
||||
|
||||
return typeof input;
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* Branching logic which calls the correct function to clone the given value base on its type.
|
||||
*/
|
||||
function cloneValue(value) {
|
||||
|
||||
// The value is an object so let's clone it.
|
||||
if (getTypeOf(value) === 'object') {
|
||||
return quickCloneObject(value);
|
||||
}
|
||||
|
||||
// The value is an array so let's clone it.
|
||||
else if (getTypeOf(value) === 'array') {
|
||||
return quickCloneArray(value);
|
||||
}
|
||||
|
||||
// Any other value can just be copied.
|
||||
return value;
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* Enumerates the given array and returns a new array, with each of its values cloned (i.e. references broken).
|
||||
*/
|
||||
function quickCloneArray(input) {
|
||||
return input.map(cloneValue);
|
||||
}
|
||||
|
||||
/*
|
||||
* Enumerates the properties of the given object (ignoring the prototype chain) and returns a new object, with each of
|
||||
* its values cloned (i.e. references broken).
|
||||
*/
|
||||
function quickCloneObject(input) {
|
||||
|
||||
const output = {};
|
||||
|
||||
for (const key in input) {
|
||||
if (!Object.prototype.hasOwnProperty.call(input, key)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
output[key] = cloneValue(input[key]);
|
||||
}
|
||||
|
||||
return output;
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* Does the actual deep merging.
|
||||
*/
|
||||
function executeDeepMerge(target, _objects = [], _options = {}) {
|
||||
|
||||
const options = {
|
||||
arrayBehaviour: _options.arrayBehaviour || 'replace', // Can be "merge" or "replace".
|
||||
};
|
||||
|
||||
// Ensure we have actual objects for each.
|
||||
const objects = _objects.map(object => object || {});
|
||||
const output = target || {};
|
||||
|
||||
// Enumerate the objects and their keys.
|
||||
for (let oindex = 0; oindex < objects.length; oindex++) {
|
||||
const object = objects[oindex];
|
||||
const keys = Object.keys(object);
|
||||
|
||||
for (let kindex = 0; kindex < keys.length; kindex++) {
|
||||
const key = keys[kindex];
|
||||
const value = object[key];
|
||||
const type = getTypeOf(value);
|
||||
const existingValueType = getTypeOf(output[key]);
|
||||
|
||||
if (type === 'object') {
|
||||
if (existingValueType !== 'undefined') {
|
||||
const existingValue = (existingValueType === 'object' ? output[key] : {});
|
||||
output[key] = executeDeepMerge({}, [existingValue, quickCloneObject(value),], options);
|
||||
} else {
|
||||
output[key] = quickCloneObject(value);
|
||||
}
|
||||
} else if (type === 'array') {
|
||||
if (existingValueType === 'array') {
|
||||
const newValue = quickCloneArray(value);
|
||||
output[key] = (options.arrayBehaviour === 'merge' ? output[key].concat(newValue) : newValue);
|
||||
} else {
|
||||
output[key] = quickCloneArray(value);
|
||||
}
|
||||
} else {
|
||||
output[key] = value;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
|
||||
}
|
||||
|
||||
export function objectAssignDeep(target, ...objects) {
|
||||
return executeDeepMerge(target, objects);
|
||||
}
|
||||
21
frontend/src/router.js
Normal file
@ -0,0 +1,21 @@
|
||||
import Vue from 'vue';
|
||||
import Router from 'vue-router';
|
||||
import ContentStream from './views/ContentStream';
|
||||
import Sidebar from './components/Sidebar';
|
||||
|
||||
Vue.use(Router);
|
||||
|
||||
export default new Router({
|
||||
routes: [
|
||||
{
|
||||
path: '/:servicePort?/:streamId?',
|
||||
name: 'stream',
|
||||
props: true,
|
||||
components: {
|
||||
sidebar: Sidebar,
|
||||
content: ContentStream,
|
||||
},
|
||||
},
|
||||
],
|
||||
linkActiveClass: 'active',
|
||||
});
|
||||
89
frontend/src/store.js
Normal file
@ -0,0 +1,89 @@
|
||||
import Vue from 'vue';
|
||||
import Vuex from 'vuex';
|
||||
import createMutationsSharer from 'vuex-shared-mutations';
|
||||
import createPersistedState from 'vuex-persistedstate';
|
||||
|
||||
Vue.use(Vuex);
|
||||
|
||||
// noinspection JSUnusedLocalSymbols,JSUnusedGlobalSymbols
|
||||
export default new Vuex.Store({
|
||||
state: {
|
||||
theme: null,
|
||||
|
||||
hexdumpBlockSize: 16,
|
||||
hexdumpLineNumberBase: 10,
|
||||
pageSize: 25,
|
||||
displayFavoritesOnly: false,
|
||||
pause: false,
|
||||
pcapStarted: true,
|
||||
|
||||
hexdumpMode: false,
|
||||
|
||||
serviceModalName: '',
|
||||
serviceModalId: 0,
|
||||
|
||||
patterns: [],
|
||||
services: [],
|
||||
|
||||
currentPacketsCount: 0,
|
||||
currentStreamsCount: 0,
|
||||
currentServicesPacketsCount: {},
|
||||
currentServicesStreamsCount: {},
|
||||
},
|
||||
mutations: {
|
||||
setTheme: (s, p) => s.theme = p,
|
||||
setHexdumpBlockSize: (s, p) => s.hexdumpBlockSize = p,
|
||||
setHexdumpLineNumberBase: (s, p) => s.hexdumpLineNumberBase = p,
|
||||
setPageSize: (s, p) => s.pageSize = p,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
toggleDisplayFavoritesOnly: (s, p) => s.displayFavoritesOnly = !s.displayFavoritesOnly,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
togglePause: (s, p) => s.pause = !s.pause,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
startPcap: (s, p) => s.pcapStarted = true,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
toggleHexdumpMode: (s, p) => s.hexdumpMode = !s.hexdumpMode,
|
||||
|
||||
setServiceModalName: (s, p) => s.serviceModalName = p,
|
||||
setServiceModalId: (s, p) => s.serviceModalId = p,
|
||||
|
||||
setPatterns: (s, p) => s.patterns = p,
|
||||
addPattern: (s, p) => s.patterns.push(p),
|
||||
|
||||
setServices: (s, p) => s.services = p,
|
||||
addService: (s, p) => s.services.push(p),
|
||||
|
||||
setCurrentPacketsCount: (s, p) => s.currentPacketsCount = p,
|
||||
setCurrentStreamsCount: (s, p) => s.currentStreamsCount = p,
|
||||
setCurrentServicesPacketsCount: (s, p) => s.currentServicesPacketsCount = p,
|
||||
setCurrentServicesStreamsCount: (s, p) => s.currentServicesStreamsCount = p,
|
||||
},
|
||||
actions: {},
|
||||
plugins: [
|
||||
createPersistedState(),
|
||||
createMutationsSharer({
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
predicate: (mutation, state) => {
|
||||
console.debug('Got mutation:', mutation);
|
||||
const mName = mutation?.type;
|
||||
const process = mName !== 'toggleDisplayFavoritesOnly'
|
||||
&& mName !== 'setTheme'
|
||||
&& mName !== 'setServiceModalName'
|
||||
&& mName !== 'setServiceModalId'
|
||||
&& mName !== 'togglePause'
|
||||
&& mName !== 'startPcap'
|
||||
&& mName !== 'toggleHexdumpMode'
|
||||
&& mName !== 'setPatterns'
|
||||
&& mName !== 'addPattern'
|
||||
&& mName !== 'setServices'
|
||||
&& mName !== 'addService'
|
||||
&& mName !== 'setCurrentPacketsCount'
|
||||
&& mName !== 'setCurrentStreamsCount'
|
||||
&& mName !== 'setCurrentServicesPacketsCount'
|
||||
&& mName !== 'setCurrentServicesStreamsCount';
|
||||
console.debug('Processing?', process);
|
||||
return process;
|
||||
},
|
||||
}),
|
||||
],
|
||||
});
|
||||
121
frontend/src/views/ContentStream.vue
Normal file
@ -0,0 +1,121 @@
|
||||
<template>
|
||||
<div>
|
||||
<p>
|
||||
<span style="background: var(--request-packet-color); box-shadow: 0 0 3px 3px var(--request-packet-color);">Request</span>,
|
||||
<span style="background: var(--response-packet-color); box-shadow: 0 0 3px 3px var(--response-packet-color);">response</span>.
|
||||
<ThemeButton class="d-inline float-right" style="margin-top: -0.25rem" />
|
||||
</p>
|
||||
<Packet v-for="packetWithOffset in packetsWithOffsets"
|
||||
:key="packetWithOffset.packet.id"
|
||||
:packet="packetWithOffset.packet"
|
||||
:offset="packetWithOffset.offset" />
|
||||
<infinite-loading @infinite="infiniteLoadingHandler" ref="infiniteLoader">
|
||||
<span slot="no-results"></span>
|
||||
</infinite-loading>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script>
|
||||
import Packet from '../components/Packet';
|
||||
import ThemeButton from '@/components/ThemeButton.vue';
|
||||
|
||||
export default {
|
||||
name: 'ContentStream',
|
||||
props: ['servicePort', 'streamId',],
|
||||
data() {
|
||||
return {
|
||||
packets: [],
|
||||
requestGeneration: 0,
|
||||
};
|
||||
},
|
||||
computed: {
|
||||
packetsWithOffsets: function() {
|
||||
return this.packets.map((el, i) => {
|
||||
let offset = null;
|
||||
|
||||
if (i !== 0) {
|
||||
offset = el.timestamp - this.packets[i - 1].timestamp;
|
||||
}
|
||||
|
||||
return {packet: el, offset: offset,}
|
||||
})
|
||||
},
|
||||
},
|
||||
watch: {
|
||||
'$route.params.streamId': function () {
|
||||
this.requestGeneration++;
|
||||
this.packets = [];
|
||||
this.$refs.infiniteLoader.stateChanger.reset();
|
||||
},
|
||||
},
|
||||
methods: {
|
||||
infiniteLoadingHandler($state) {
|
||||
if (!this.$route.params.streamId) return $state.complete();
|
||||
const generation = this.requestGeneration;
|
||||
const streamId = this.$route.params.streamId;
|
||||
|
||||
const packets = this.packets;
|
||||
const pageSize = this.$store.state.pageSize;
|
||||
let startsFrom;
|
||||
if (packets && packets.length && packets[packets.length - 1]) {
|
||||
startsFrom = packets[packets.length - 1].id;
|
||||
} else {
|
||||
startsFrom = null;
|
||||
}
|
||||
|
||||
this.$http.post(`packet/${streamId}`, {
|
||||
startingFrom: startsFrom,
|
||||
pageSize: pageSize,
|
||||
}).then(response => {
|
||||
if (generation !== this.requestGeneration || streamId !== this.$route.params.streamId) return;
|
||||
const data = response.data;
|
||||
if (data.length === 0) {
|
||||
console.log('Finished loading packets (empty page)');
|
||||
return $state.complete();
|
||||
}
|
||||
|
||||
if (data[0] && this.packets[0] && data[0].id === this.packets[0].id) {
|
||||
console.log('Finished loading packets (overlap detected)');
|
||||
return $state.complete();
|
||||
}
|
||||
|
||||
this.packets.push(...data);
|
||||
|
||||
if (data.length < pageSize) {
|
||||
// this was the last page
|
||||
console.log('Finished loading packets (last page was not full)');
|
||||
$state.complete();
|
||||
} else {
|
||||
console.log('Loaded another page of packets');
|
||||
$state.loaded();
|
||||
}
|
||||
}).catch(e => {
|
||||
if (generation !== this.requestGeneration || streamId !== this.$route.params.streamId) return;
|
||||
this.$bvToast.toast(`Failed to load portion of packets: ${e}`, {
|
||||
title: 'Error',
|
||||
variant: 'danger',
|
||||
});
|
||||
console.error('Failed to load portion of packets:', e);
|
||||
return $state.error();
|
||||
});
|
||||
},
|
||||
},
|
||||
components: {
|
||||
ThemeButton,
|
||||
Packet,
|
||||
},
|
||||
};
|
||||
</script>
|
||||
|
||||
<style>
|
||||
:root {
|
||||
--response-packet-color: rgb(231, 248, 253);
|
||||
--request-packet-color: rgb(254, 234, 231);
|
||||
}
|
||||
|
||||
:root [data-theme=dark] {
|
||||
color-scheme: dark;
|
||||
--response-packet-color: rgb(43, 83, 120);
|
||||
--request-packet-color: rgb(52, 58, 64);
|
||||
}
|
||||
</style>
|
||||
57
frontend/src/views/LookBack.vue
Normal file
@ -0,0 +1,57 @@
|
||||
<template>
|
||||
<b-modal @ok="lookBack" id="lookBackModal"
|
||||
title="Search into the past"
|
||||
cancel-title="Cancel"
|
||||
centered scrollable>
|
||||
<form ref="lookBackForm">
|
||||
<b-form-group
|
||||
label-cols-sm="4"
|
||||
label-cols-lg="3"
|
||||
label="Minutes"
|
||||
description="How far into the past do we want to look"
|
||||
label-for="lookback-minutes">
|
||||
<b-form-input @keydown.native.enter="lookBack" id="lookback-minutes" required v-model="minutes"/>
|
||||
</b-form-group>
|
||||
</form>
|
||||
</b-modal>
|
||||
</template>
|
||||
|
||||
<script>
|
||||
export default {
|
||||
name: 'LookBack',
|
||||
props: {
|
||||
patternId: Number,
|
||||
},
|
||||
data() {
|
||||
return {
|
||||
minutes: 5,
|
||||
};
|
||||
},
|
||||
methods: {
|
||||
checkValidity() {
|
||||
return this.$refs.lookBackForm.reportValidity();
|
||||
},
|
||||
|
||||
lookBack(ev) {
|
||||
ev.preventDefault();
|
||||
if (!this.checkValidity()) {
|
||||
console.debug('Form is invalid');
|
||||
return;
|
||||
}
|
||||
console.debug(`Looking back with pattern ${this.patternId}`);
|
||||
|
||||
this.$http.post(`pattern/${this.patternId}/lookback`, this.minutes)
|
||||
.then(() => {
|
||||
console.debug('Lookback started');
|
||||
this.$bvModal.hide('lookBackModal');
|
||||
}).catch(e => {
|
||||
this.$bvToast.toast(`Failed to start lookback: ${e}`, {
|
||||
title: 'Error',
|
||||
variant: 'danger',
|
||||
});
|
||||
console.error('Failed to start lookback', e);
|
||||
});
|
||||
},
|
||||
},
|
||||
};
|
||||
</script>
|
||||
226
frontend/src/views/PatternModal.vue
Normal file
@ -0,0 +1,226 @@
|
||||
<template>
|
||||
<b-modal @ok="addPattern" @cancel="reset" id="patternModal"
|
||||
:title="creating ? 'Creating pattern' : 'Editing pattern'"
|
||||
cancel-title="Cancel"
|
||||
centered scrollable>
|
||||
<form ref="addPatternForm">
|
||||
<b-form-group
|
||||
label-cols-sm="4"
|
||||
label-cols-lg="3"
|
||||
label="Name"
|
||||
description="It is displayed in the list and highlights matching content or IP addresses"
|
||||
label-for="pattern-name">
|
||||
<b-form-input @keydown.native.enter="addPattern" id="pattern-name" required v-model="newPattern.name"/>
|
||||
</b-form-group>
|
||||
|
||||
<b-form-group
|
||||
v-if="creating"
|
||||
label-cols-sm="4"
|
||||
label-cols-lg="3"
|
||||
label="Pattern"
|
||||
description="Substring, RegEx or bytes"
|
||||
label-for="pattern-value">
|
||||
<b-form-input @keydown.native.enter="addPattern" @keydown="validateKey"
|
||||
id="pattern-value" required v-model="newPattern.value"
|
||||
:placeholder="getPlaceholder()"/>
|
||||
</b-form-group>
|
||||
|
||||
<b-form-group
|
||||
v-if="creating"
|
||||
label-cols-sm="4"
|
||||
label-cols-lg="3"
|
||||
label="Search action"
|
||||
description="What to do with matching streams"
|
||||
label-for="pattern-actionType">
|
||||
<b-form-select id="pattern-actionType" required v-model="newPattern.actionType">
|
||||
<option value="FIND" selected>Highlight found pattern</option>
|
||||
<option value="IGNORE">Ignore matching streams</option>
|
||||
</b-form-select>
|
||||
</b-form-group>
|
||||
|
||||
<b-form-group v-if="newPattern.actionType === 'FIND'"
|
||||
label-cols-sm="4"
|
||||
label-cols-lg="3"
|
||||
label="Color"
|
||||
description="The highlight color of the pattern in the packets and streams"
|
||||
label-for="pattern-color">
|
||||
<b-form-input id="pattern-color" required type="color" v-model="newPattern.color"/>
|
||||
</b-form-group>
|
||||
|
||||
<b-form-group
|
||||
v-if="creating"
|
||||
label-cols-sm="4"
|
||||
label-cols-lg="3"
|
||||
label="Search in"
|
||||
description="IP addresses use regular expressions. Use ^ and $ to match the entire address."
|
||||
label-for="pattern-searchTarget">
|
||||
<b-form-select id="pattern-searchTarget" required v-model="newPattern.searchTarget" @change="onSearchTargetChange">
|
||||
<option value="CONTENT">Packet content</option>
|
||||
<option value="SOURCE_IP">Source IP</option>
|
||||
<option value="DESTINATION_IP">Destination IP</option>
|
||||
<option value="EITHER_IP">Either IP address</option>
|
||||
</b-form-select>
|
||||
</b-form-group>
|
||||
|
||||
<b-form-group
|
||||
v-if="creating"
|
||||
label-cols-sm="4"
|
||||
label-cols-lg="3"
|
||||
label="Search method"
|
||||
description="The way to search for patterns in packets"
|
||||
label-for="pattern-searchType">
|
||||
<b-form-select id="pattern-searchType" required v-model="newPattern.searchType" :disabled="newPattern.searchTarget !== 'CONTENT'">
|
||||
<option value="REGEX" selected>Regular expression</option>
|
||||
<option value="SUBSTRING">Substring</option>
|
||||
<option value="SUBBYTES">Bytes</option>
|
||||
</b-form-select>
|
||||
</b-form-group>
|
||||
|
||||
<b-form-group
|
||||
v-if="creating"
|
||||
label-cols-sm="4"
|
||||
label-cols-lg="3"
|
||||
label="Search type"
|
||||
description="In which packets to search for a pattern"
|
||||
label-for="pattern-type">
|
||||
<b-form-select id="pattern-type" required v-model="newPattern.directionType">
|
||||
<option value="INPUT">Request</option>
|
||||
<option value="OUTPUT">Response</option>
|
||||
<option value="BOTH" selected>Anywhere</option>
|
||||
</b-form-select>
|
||||
</b-form-group>
|
||||
|
||||
<b-form-group
|
||||
v-if="creating"
|
||||
label-cols-sm="4"
|
||||
label-cols-lg="3"
|
||||
label="Service"
|
||||
description="Apply this pattern only to the specific service"
|
||||
label-for="pattern-service">
|
||||
<b-form-select id="pattern-service" :options="serviceOptions"
|
||||
v-model="newPattern.serviceId"></b-form-select>
|
||||
</b-form-group>
|
||||
</form>
|
||||
</b-modal>
|
||||
</template>
|
||||
|
||||
<script>
|
||||
|
||||
const hexRegex = /[0-9A-Fa-f ]/;
|
||||
const defaultPattern = {
|
||||
name: '',
|
||||
value: '',
|
||||
color: '#FF7474',
|
||||
searchType: 'SUBSTRING',
|
||||
searchTarget: 'CONTENT',
|
||||
directionType: 'BOTH',
|
||||
actionType: 'FIND',
|
||||
serviceId: null,
|
||||
};
|
||||
|
||||
export default {
|
||||
name: 'PatternModal',
|
||||
props: {
|
||||
creating: Boolean,
|
||||
|
||||
initialPattern: {
|
||||
name: String,
|
||||
value: String,
|
||||
color: String,
|
||||
searchType: String,
|
||||
searchTarget: String,
|
||||
directionType: String,
|
||||
actionType: String,
|
||||
serviceId: Number,
|
||||
},
|
||||
},
|
||||
data() {
|
||||
return {
|
||||
newPattern: {},
|
||||
};
|
||||
},
|
||||
watch: {
|
||||
initialPattern() {
|
||||
console.debug('initialService changed, reassigning...', this.initialPattern);
|
||||
this.newPattern = {...defaultPattern, ...this.initialPattern,};
|
||||
},
|
||||
},
|
||||
computed: {
|
||||
serviceOptions: function () {
|
||||
let services = this.$store.state.services;
|
||||
let options = services.map(service => {
|
||||
return {
|
||||
value: service.port,
|
||||
text: `${service.name} #${service.port}`,
|
||||
}
|
||||
});
|
||||
|
||||
return [
|
||||
{ value: null, text: 'Any service', },
|
||||
...options,
|
||||
];
|
||||
},
|
||||
},
|
||||
methods: {
|
||||
onSearchTargetChange(target) {
|
||||
if (target !== 'CONTENT') this.newPattern.searchType = 'REGEX';
|
||||
},
|
||||
getPlaceholder() {
|
||||
if (this.newPattern.searchTarget && this.newPattern.searchTarget !== 'CONTENT') return '^10\\.20\\.30\\.[0-9]+$';
|
||||
if (this.newPattern.searchType === 'REGEX') return '[A-Z0-9]{31}=';
|
||||
else if (this.newPattern.searchType === 'SUBSTRING') return 'HTTP/2';
|
||||
else return 'DEAD BEEF 1337';
|
||||
},
|
||||
validateKey(e) {
|
||||
console.log('', e);
|
||||
if (this.newPattern.searchType !== 'SUBBYTES') return;
|
||||
if (!hexRegex.test(e?.key)) {
|
||||
e.preventDefault();
|
||||
}
|
||||
},
|
||||
|
||||
checkValidity() {
|
||||
return this.$refs.addPatternForm.reportValidity();
|
||||
},
|
||||
|
||||
reset() {
|
||||
this.newPattern = {...defaultPattern,};
|
||||
},
|
||||
|
||||
addPattern(ev) {
|
||||
ev.preventDefault();
|
||||
if (!this.checkValidity()) {
|
||||
console.debug('Form is invalid');
|
||||
return;
|
||||
}
|
||||
console.debug('Adding/editing pattern...', this.newPattern);
|
||||
|
||||
if (this.newPattern.searchType === 'SUBBYTES') {
|
||||
this.newPattern.value = this.newPattern.value.replace(/\s+/g, '').toLowerCase();
|
||||
}
|
||||
|
||||
let url;
|
||||
if (this.creating) {
|
||||
url = 'pattern/'
|
||||
} else {
|
||||
url = 'pattern/' + this.newPattern.id
|
||||
}
|
||||
|
||||
this.$http.post(url, this.newPattern)
|
||||
.then(response => {
|
||||
const data = response.data;
|
||||
console.debug('Done adding/editing pattern', data);
|
||||
this.$emit('patternAddComplete');
|
||||
this.reset();
|
||||
this.$bvModal.hide('patternModal');
|
||||
}).catch(e => {
|
||||
this.$bvToast.toast(`Failed to add pattern: ${e}`, {
|
||||
title: 'Error',
|
||||
variant: 'danger',
|
||||
});
|
||||
console.error('Failed to add pattern', e);
|
||||
});
|
||||
},
|
||||
},
|
||||
};
|
||||
</script>
|
||||
136
frontend/src/views/ServiceModal.vue
Normal file
@ -0,0 +1,136 @@
|
||||
<template>
|
||||
<b-modal id="serviceModal" size="lg"
|
||||
:title="creating ? 'Creating service' : 'Editing service'"
|
||||
centered scrollable @ok.prevent="submit">
|
||||
<b-form ref="serviceForm">
|
||||
<b-form-group label-cols-sm="4"
|
||||
label="Name"
|
||||
label-for="service-name">
|
||||
<b-form-input id="service-name" required v-model="service.name"
|
||||
@keyup.enter.stop.prevent="submit"/>
|
||||
</b-form-group>
|
||||
|
||||
<b-form-group v-if="creating"
|
||||
label-cols-sm="4"
|
||||
label="Port"
|
||||
label-for="service-port">
|
||||
<b-form-input id="service-port" required
|
||||
type="number" min="1" max="65535" v-model.number="service.port"
|
||||
@keyup.enter.stop.prevent="submit"/>
|
||||
</b-form-group>
|
||||
|
||||
<b-form-group label-cols-sm="4"
|
||||
label="Is an HTTP service"
|
||||
label-for="service-is-http">
|
||||
<b-form-checkbox id="service-is-http" required
|
||||
v-model="service.http" />
|
||||
</b-form-group>
|
||||
|
||||
<b-form-group label-cols-sm="4"
|
||||
label="Apply urldecode"
|
||||
label-for="service-urldecode">
|
||||
<b-form-checkbox id="service-urldecode" required
|
||||
v-model="service.urldecodeHttpRequests"/>
|
||||
</b-form-group>
|
||||
|
||||
<b-form-group label-cols-sm="4"
|
||||
label="Merge adjacent packets"
|
||||
label-for="service-mergeAdjacent">
|
||||
<b-form-checkbox id="service-mergeAdjacent" required
|
||||
v-model="service.mergeAdjacentPackets"/>
|
||||
</b-form-group>
|
||||
|
||||
<b-form-group label-cols-sm="4"
|
||||
label="Inflate WebSockets"
|
||||
label-for="service-inflateWS">
|
||||
<b-form-checkbox id="service-inflateWS" required
|
||||
v-model="service.parseWebSockets"/>
|
||||
</b-form-group>
|
||||
|
||||
<b-form-group label-cols-sm="4"
|
||||
label="Decrypt TLS (TLS_RSA_WITH_AES only)"
|
||||
label-for="service-decryptTls">
|
||||
<b-form-checkbox id="service-decryptTls" required
|
||||
v-model="service.decryptTls"/>
|
||||
</b-form-group>
|
||||
|
||||
<b-button v-if="!creating" variant="danger" @click="deleteService">Delete</b-button>
|
||||
</b-form>
|
||||
</b-modal>
|
||||
</template>
|
||||
|
||||
<script>
|
||||
import {objectAssignDeep,} from '@/objectAssignDeep';
|
||||
|
||||
export default {
|
||||
name: 'ServiceModal',
|
||||
props: {
|
||||
creating: Boolean,
|
||||
|
||||
initialService: {
|
||||
name: String,
|
||||
port: Number,
|
||||
http: Boolean,
|
||||
urldecodeHttpRequests: Boolean,
|
||||
mergeAdjacentPackets: Boolean,
|
||||
parseWebSockets: Boolean,
|
||||
decryptTls: Boolean,
|
||||
},
|
||||
},
|
||||
data() {
|
||||
return {
|
||||
service: {},
|
||||
};
|
||||
},
|
||||
watch: {
|
||||
initialService() {
|
||||
console.debug('initialService changed, reassigning...', this.initialService);
|
||||
this.service = objectAssignDeep({}, this.initialService);
|
||||
},
|
||||
},
|
||||
methods: {
|
||||
submit() {
|
||||
if (!this.$refs.serviceForm.reportValidity()) {
|
||||
return;
|
||||
}
|
||||
console.debug('Submitting service...', this.service, this.creating);
|
||||
|
||||
let url;
|
||||
|
||||
if (this.creating) {
|
||||
url = 'service/';
|
||||
} else {
|
||||
url = 'service/' + this.initialService.port
|
||||
}
|
||||
|
||||
this.$http.post(url, this.service)
|
||||
.then(response => {
|
||||
console.info('Done editing/creating service', response.data);
|
||||
this.$emit('service-update-needed');
|
||||
this.$bvModal.hide('serviceModal');
|
||||
})
|
||||
.catch(e => {
|
||||
this.$bvToast.toast(`Failed to ${this.creating ? 'create' : 'edit'} service: ${e}`, {
|
||||
title: 'Error',
|
||||
variant: 'danger',
|
||||
});
|
||||
console.error('Failed to edit/create service', e);
|
||||
});
|
||||
},
|
||||
deleteService() {
|
||||
this.$http.delete(`service/${this.service.port}`)
|
||||
.then(() => {
|
||||
console.info('Done deleting service', this.service);
|
||||
this.$emit('service-update-needed');
|
||||
this.$bvModal.hide('serviceModal');
|
||||
}).catch(e => {
|
||||
this.$bvToast.toast(`Failed to delete service: ${e}`, {
|
||||
title: 'Error',
|
||||
variant: 'danger',
|
||||
});
|
||||
console.error('Failed to delete service', e);
|
||||
});
|
||||
},
|
||||
},
|
||||
};
|
||||
</script>
|
||||
63
frontend/src/views/Settings.vue
Normal file
@ -0,0 +1,63 @@
|
||||
<template>
|
||||
<b-modal id="settingsModal" title="Settings" centered scrollable ok-only>
|
||||
<b-form-group
|
||||
label-cols-sm="4"
|
||||
label-cols-lg="3"
|
||||
description="The number of bytes to display in the binary representation of the packet"
|
||||
label="HEX block width"
|
||||
label-for="settings-hexdumpBlockSize">
|
||||
<b-form-input type="number" id="settings-hexdumpBlockSize" v-model.number="hexdumpBlockSize"/>
|
||||
</b-form-group>
|
||||
<b-form-group
|
||||
label-cols-sm="4"
|
||||
label-cols-lg="3"
|
||||
description="The number system used in line numbers of a binary representation of a packet"
|
||||
label="Line numbering"
|
||||
label-for="settings-hexdumpLineNumberBase">
|
||||
<b-form-select id="settings-hexdumpLineNumberBase" v-model="hexdumpLineNumberBase">
|
||||
<option :value="10" selected>Decimal</option>
|
||||
<option :value="16">Hexadecimal</option>
|
||||
</b-form-select>
|
||||
</b-form-group>
|
||||
<b-form-group
|
||||
label-cols-sm="4"
|
||||
label-cols-lg="3"
|
||||
description="The number of streams to download at a time"
|
||||
label="Page size"
|
||||
label-for="settings-pageSize">
|
||||
<b-form-input type="number" id="settings-pageSize" v-model.number="pageSize"/>
|
||||
</b-form-group>
|
||||
</b-modal>
|
||||
</template>
|
||||
|
||||
<script>
|
||||
export default {
|
||||
name: 'Settings',
|
||||
computed: {
|
||||
hexdumpBlockSize: {
|
||||
get() {
|
||||
return this.$store.state.hexdumpBlockSize;
|
||||
},
|
||||
set(v) {
|
||||
this.$store.commit('setHexdumpBlockSize', v);
|
||||
},
|
||||
},
|
||||
hexdumpLineNumberBase: {
|
||||
get() {
|
||||
return this.$store.state.hexdumpLineNumberBase;
|
||||
},
|
||||
set(v) {
|
||||
this.$store.commit('setHexdumpLineNumberBase', v);
|
||||
},
|
||||
},
|
||||
pageSize: {
|
||||
get() {
|
||||
return this.$store.state.pageSize;
|
||||
},
|
||||
set(v) {
|
||||
this.$store.commit('setPageSize', v);
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
</script>
|
||||
79
frontend/tests/ip-patterns.test.cjs
Normal file
@ -0,0 +1,79 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const vm = require('node:vm');
|
||||
const test = require('node:test');
|
||||
const compiler = require('vue-template-compiler');
|
||||
|
||||
// Exercise the actual component methods without a browser or an extra test framework.
|
||||
function component(relativePath) {
|
||||
const file = fs.readFileSync(path.join(__dirname, '../src', relativePath), 'utf8');
|
||||
const source = compiler.parseComponent(file).script.content
|
||||
.replace(/^\s*import .*$/gm, '')
|
||||
.replace('export default', 'module.exports =');
|
||||
const sandbox = {module: {exports: {}}, SidebarStream: {}, PatternModal: {}, LookBack: {}, console};
|
||||
vm.runInNewContext(source, sandbox);
|
||||
return sandbox.module.exports;
|
||||
}
|
||||
|
||||
test('IP selection uses regex and provides an escaped IPv4 example', () => {
|
||||
const methods = component('views/PatternModal.vue').methods;
|
||||
const context = {newPattern: {searchTarget: 'SOURCE_IP', searchType: 'SUBBYTES'}};
|
||||
methods.onSearchTargetChange.call(context, 'SOURCE_IP');
|
||||
assert.equal(context.newPattern.searchType, 'REGEX');
|
||||
assert.equal(methods.getPlaceholder.call(context), '^10\\.20\\.30\\.[0-9]+$');
|
||||
});
|
||||
|
||||
test('live filter accepts matching DTO IDs and rejects other streams', () => {
|
||||
const addStream = component('components/Sidebar.vue').methods.addStreamFromWs;
|
||||
const context = {
|
||||
$route: {params: {servicePort: '80'}, query: {pattern: '7'}},
|
||||
$store: {state: {displayFavoritesOnly: false, pause: false}},
|
||||
streams: [],
|
||||
};
|
||||
addStream.call(context, {id: 1, service: 80, foundPatternsIds: [7]});
|
||||
addStream.call(context, {id: 2, service: 80, foundPatternsIds: [8]});
|
||||
addStream.call(context, {id: 3, service: 443, foundPatternsIds: [7]});
|
||||
addStream.call(context, {id: 4, service: 80});
|
||||
assert.deepEqual(context.streams.map(stream => stream.id), [1]);
|
||||
});
|
||||
|
||||
test('IP matches highlight endpoints without changing payload highlights', () => {
|
||||
const methods = component('components/Packet.vue').methods;
|
||||
const context = {
|
||||
$store: {state: {patterns: [
|
||||
{id: 1, color: '#112233', deleted: false},
|
||||
{id: 2, color: '#445566', deleted: false},
|
||||
]}},
|
||||
packet: {matches: [
|
||||
{patternId: 1, searchTarget: 'SOURCE_IP', startPosition: 0, endPosition: 10},
|
||||
{patternId: 2, searchTarget: 'CONTENT', startPosition: 0, endPosition: 3},
|
||||
]},
|
||||
};
|
||||
assert.equal(methods.ipHighlightStyle.call(context, 'SOURCE_IP').backgroundColor, '#112233');
|
||||
assert.equal(Object.keys(methods.ipHighlightStyle.call(context, 'DESTINATION_IP')).length, 0);
|
||||
assert.equal(methods.highlightPatterns.call(context, 'FLAG rest'),
|
||||
'<span style="background-color: #445566">FLAG</span> rest');
|
||||
|
||||
context.packet.matches = [context.packet.matches[0]];
|
||||
assert.equal(methods.highlightPatterns.call(context, 'unchanged'), 'unchanged');
|
||||
context.$store.state.patterns[0].deleted = true;
|
||||
assert.equal(Object.keys(methods.ipHighlightStyle.call(context, 'SOURCE_IP')).length, 0);
|
||||
});
|
||||
|
||||
test('legacy content matches without a target retain their highlights', () => {
|
||||
const methods = component('components/Packet.vue').methods;
|
||||
const context = {
|
||||
$store: {state: {patterns: [{id: 1, color: '#112233', deleted: false}]}},
|
||||
packet: {matches: [{patternId: 1, startPosition: 0, endPosition: 3}]},
|
||||
};
|
||||
assert.equal(methods.highlightPatterns.call(context, 'FLAG'),
|
||||
'<span style="background-color: #112233">FLAG</span>');
|
||||
});
|
||||
|
||||
test('pattern list identifies the selected IP field', () => {
|
||||
const method = component('components/PatternsDropdown.vue').methods.getSearchTargetLabel;
|
||||
assert.equal(method('SOURCE_IP'), 'source IP');
|
||||
assert.equal(method('DESTINATION_IP'), 'destination IP');
|
||||
assert.equal(method('EITHER_IP'), 'either IP address');
|
||||
});
|
||||
3
frontend/vue.config.js
Normal file
@ -0,0 +1,3 @@
|
||||
module.exports = {
|
||||
lintOnSave: false,
|
||||
};
|
||||
@ -10,6 +10,8 @@ import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.scheduling.annotation.EnableAsync;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
|
||||
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
|
||||
import ru.serega6531.packmate.model.Pattern;
|
||||
import ru.serega6531.packmate.model.Stream;
|
||||
import ru.serega6531.packmate.model.pojo.StreamDto;
|
||||
@ -24,6 +26,7 @@ import ru.serega6531.packmate.service.SubscriptionService;
|
||||
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Configuration
|
||||
@ -32,6 +35,30 @@ import java.util.stream.Collectors;
|
||||
@ConfigurationPropertiesScan("ru.serega6531.packmate.properties")
|
||||
public class ApplicationConfiguration {
|
||||
|
||||
@Bean
|
||||
public ThreadPoolTaskScheduler taskScheduler() {
|
||||
ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler();
|
||||
scheduler.setPoolSize(4);
|
||||
scheduler.setThreadNamePrefix("packmate-scheduled-");
|
||||
scheduler.setRemoveOnCancelPolicy(true);
|
||||
scheduler.setWaitForTasksToCompleteOnShutdown(true);
|
||||
scheduler.setAwaitTerminationSeconds(10);
|
||||
return scheduler;
|
||||
}
|
||||
|
||||
@Bean(name = "lookbackExecutor")
|
||||
public ThreadPoolTaskExecutor lookbackExecutor() {
|
||||
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
|
||||
executor.setCorePoolSize(1);
|
||||
executor.setMaxPoolSize(1);
|
||||
executor.setQueueCapacity(2);
|
||||
executor.setThreadNamePrefix("packmate-lookback-");
|
||||
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
|
||||
executor.setWaitForTasksToCompleteOnShutdown(true);
|
||||
executor.setAwaitTerminationSeconds(10);
|
||||
return executor;
|
||||
}
|
||||
|
||||
@Bean(destroyMethod = "stop")
|
||||
@Autowired
|
||||
public PcapWorker pcapWorker(ServicesService servicesService,
|
||||
@ -40,7 +67,8 @@ public class ApplicationConfiguration {
|
||||
PackmateProperties properties
|
||||
) throws PcapNativeException, UnknownHostException {
|
||||
return switch (properties.captureMode()) {
|
||||
case LIVE -> new LivePcapWorker(servicesService, streamService, properties.localIp(), properties.interfaceName());
|
||||
case LIVE -> new LivePcapWorker(servicesService, streamService, properties.localIp(),
|
||||
properties.interfaceName(), properties.capture());
|
||||
case FILE ->
|
||||
new FilePcapWorker(servicesService, streamService, subscriptionService, properties.localIp(), properties.pcapFile());
|
||||
case VIEW -> new NoOpPcapWorker();
|
||||
|
||||
@ -0,0 +1,192 @@
|
||||
package ru.serega6531.packmate.configuration;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.boot.ApplicationArguments;
|
||||
import org.springframework.boot.ApplicationRunner;
|
||||
import org.springframework.core.Ordered;
|
||||
import org.springframework.core.annotation.Order;
|
||||
import org.springframework.jdbc.core.ConnectionCallback;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
@Component
|
||||
@Order(Ordered.HIGHEST_PRECEDENCE)
|
||||
@Slf4j
|
||||
public class CleanupDatabaseInitializer implements ApplicationRunner {
|
||||
|
||||
private static final long INITIALIZER_LOCK_ID = 5_788_337_447_881_982_277L;
|
||||
private static final List<IndexSpec> INDEXES = List.of(
|
||||
new IndexSpec(
|
||||
"stream_cleanup_index",
|
||||
"stream",
|
||||
"end_timestamp,id",
|
||||
"favorite=falseandcleanup_pending=false",
|
||||
"""
|
||||
CREATE INDEX CONCURRENTLY stream_cleanup_index
|
||||
ON stream (end_timestamp, id)
|
||||
WHERE favorite = false AND cleanup_pending = false
|
||||
"""
|
||||
),
|
||||
new IndexSpec(
|
||||
"stream_cleanup_pending_index",
|
||||
"stream",
|
||||
"cleanup_retry_after,end_timestamp,id",
|
||||
"cleanup_pending=true",
|
||||
"""
|
||||
CREATE INDEX CONCURRENTLY stream_cleanup_pending_index
|
||||
ON stream (cleanup_retry_after, end_timestamp, id)
|
||||
WHERE cleanup_pending = true
|
||||
"""
|
||||
),
|
||||
new IndexSpec(
|
||||
"packet_cleanup_stream_id_index",
|
||||
"packet",
|
||||
"stream_id,id",
|
||||
"",
|
||||
"""
|
||||
CREATE INDEX CONCURRENTLY packet_cleanup_stream_id_index
|
||||
ON packet (stream_id, id)
|
||||
"""
|
||||
)
|
||||
);
|
||||
|
||||
private final JdbcTemplate jdbc;
|
||||
|
||||
public CleanupDatabaseInitializer(JdbcTemplate jdbc) {
|
||||
this.jdbc = jdbc;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(ApplicationArguments args) {
|
||||
initializeIndexes();
|
||||
}
|
||||
|
||||
public void initializeIndexes() {
|
||||
jdbc.execute((ConnectionCallback<Void>) connection -> {
|
||||
if (!connection.getAutoCommit()) {
|
||||
throw new IllegalStateException("Cleanup indexes must be initialized outside a transaction");
|
||||
}
|
||||
|
||||
advisoryLock(connection, true);
|
||||
try {
|
||||
for (IndexSpec spec : INDEXES) {
|
||||
ensureIndex(connection, spec);
|
||||
}
|
||||
} finally {
|
||||
advisoryLock(connection, false);
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
private void ensureIndex(Connection connection, IndexSpec spec) throws SQLException {
|
||||
IndexState state = readIndexState(connection, spec.name());
|
||||
if (state != null && !spec.matches(state)) {
|
||||
log.warn("Rebuilding invalid or unexpected cleanup index {}: {}", spec.name(), state);
|
||||
execute(connection, "DROP INDEX CONCURRENTLY " + spec.name());
|
||||
state = null;
|
||||
}
|
||||
|
||||
if (state == null) {
|
||||
execute(connection, spec.createSql());
|
||||
}
|
||||
|
||||
IndexState finalState = readIndexState(connection, spec.name());
|
||||
if (finalState == null || !spec.matches(finalState)) {
|
||||
throw new IllegalStateException("Cleanup index was not created correctly: " + spec.name());
|
||||
}
|
||||
log.info("Cleanup database index {} is ready", spec.name());
|
||||
}
|
||||
|
||||
private IndexState readIndexState(Connection connection, String indexName) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement("""
|
||||
SELECT i.indisvalid,
|
||||
i.indisready,
|
||||
table_class.relname,
|
||||
access_method.amname,
|
||||
COALESCE((
|
||||
SELECT string_agg(pg_get_indexdef(i.indexrelid, position, false), ',' ORDER BY position)
|
||||
FROM generate_series(1, i.indnkeyatts) AS position
|
||||
), ''),
|
||||
COALESCE(pg_get_expr(i.indpred, i.indrelid), '')
|
||||
FROM pg_class index_class
|
||||
JOIN pg_namespace namespace ON namespace.oid = index_class.relnamespace
|
||||
JOIN pg_index i ON i.indexrelid = index_class.oid
|
||||
JOIN pg_class table_class ON table_class.oid = i.indrelid
|
||||
JOIN pg_am access_method ON access_method.oid = index_class.relam
|
||||
WHERE namespace.nspname = current_schema()
|
||||
AND index_class.relname = ?
|
||||
""")) {
|
||||
statement.setString(1, indexName);
|
||||
try (ResultSet result = statement.executeQuery()) {
|
||||
if (!result.next()) {
|
||||
return null;
|
||||
}
|
||||
return new IndexState(
|
||||
result.getBoolean(1),
|
||||
result.getBoolean(2),
|
||||
result.getString(3),
|
||||
result.getString(4),
|
||||
normalize(result.getString(5)),
|
||||
normalize(result.getString(6))
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void advisoryLock(Connection connection, boolean acquire) throws SQLException {
|
||||
String function = acquire ? "pg_advisory_lock" : "pg_advisory_unlock";
|
||||
try (PreparedStatement statement = connection.prepareStatement("SELECT " + function + "(?)")) {
|
||||
statement.setLong(1, INITIALIZER_LOCK_ID);
|
||||
statement.execute();
|
||||
}
|
||||
}
|
||||
|
||||
private void execute(Connection connection, String sql) throws SQLException {
|
||||
try (Statement statement = connection.createStatement()) {
|
||||
statement.execute(sql);
|
||||
}
|
||||
}
|
||||
|
||||
private static String normalize(String value) {
|
||||
return value == null
|
||||
? ""
|
||||
: value.toLowerCase(Locale.ROOT)
|
||||
.replace("\"", "")
|
||||
.replaceAll("[\\s()]", "");
|
||||
}
|
||||
|
||||
private record IndexSpec(
|
||||
String name,
|
||||
String table,
|
||||
String columns,
|
||||
String predicate,
|
||||
String createSql
|
||||
) {
|
||||
private boolean matches(IndexState state) {
|
||||
return state.valid()
|
||||
&& state.ready()
|
||||
&& table.equals(state.table())
|
||||
&& "btree".equals(state.accessMethod())
|
||||
&& columns.equals(state.columns())
|
||||
&& predicate.equals(state.predicate());
|
||||
}
|
||||
}
|
||||
|
||||
private record IndexState(
|
||||
boolean valid,
|
||||
boolean ready,
|
||||
String table,
|
||||
String accessMethod,
|
||||
String columns,
|
||||
String predicate
|
||||
) {}
|
||||
}
|
||||
@ -6,6 +6,7 @@ import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import ru.serega6531.packmate.model.pojo.PcapMetrics;
|
||||
import ru.serega6531.packmate.service.PcapService;
|
||||
|
||||
@RestController
|
||||
@ -29,4 +30,9 @@ public class PcapController {
|
||||
service.start();
|
||||
}
|
||||
|
||||
@GetMapping("/metrics")
|
||||
public PcapMetrics metrics() {
|
||||
return service.getMetrics();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -6,6 +6,7 @@ import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
import ru.serega6531.packmate.model.pojo.StreamDto;
|
||||
import ru.serega6531.packmate.model.pojo.StreamPagination;
|
||||
import ru.serega6531.packmate.service.StreamService;
|
||||
@ -13,6 +14,8 @@ import ru.serega6531.packmate.service.StreamService;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.springframework.http.HttpStatus.BAD_REQUEST;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/api/stream/")
|
||||
public class StreamController {
|
||||
@ -44,4 +47,15 @@ public class StreamController {
|
||||
service.setFavorite(id, false);
|
||||
}
|
||||
|
||||
@PostMapping("/available")
|
||||
public List<Long> availableStreams(@RequestBody StreamAvailabilityRequest request) {
|
||||
if (request == null || request.ids() == null || request.ids().size() > 500
|
||||
|| request.ids().stream().anyMatch(id -> id == null || id <= 0)) {
|
||||
throw new ResponseStatusException(BAD_REQUEST, "ids must contain at most 500 positive values");
|
||||
}
|
||||
return service.findAvailableStreamIds(request.ids());
|
||||
}
|
||||
|
||||
public record StreamAvailabilityRequest(List<Long> ids) {}
|
||||
|
||||
}
|
||||
|
||||
@ -30,7 +30,9 @@ import java.util.Set;
|
||||
@Parameter(name = "optimizer", value = "hilo")
|
||||
}
|
||||
)
|
||||
@Table(indexes = {@Index(name = "stream_id_desc_index", columnList = "id DESC")})
|
||||
@Table(indexes = {
|
||||
@Index(name = "stream_id_desc_index", columnList = "id DESC")
|
||||
})
|
||||
public class Stream {
|
||||
|
||||
@Id
|
||||
@ -64,6 +66,18 @@ public class Stream {
|
||||
|
||||
private boolean favorite;
|
||||
|
||||
@Column(name = "cleanup_pending", nullable = false, columnDefinition = "boolean default false")
|
||||
private boolean cleanupPending;
|
||||
|
||||
@Column(name = "persisted_at", nullable = false, columnDefinition = "bigint default 0")
|
||||
private long persistedAt;
|
||||
|
||||
@Column(name = "cleanup_retry_after", nullable = false, columnDefinition = "bigint default 0")
|
||||
private long cleanupRetryAfter;
|
||||
|
||||
@Column(name = "cleanup_attempts", nullable = false, columnDefinition = "integer default 0")
|
||||
private int cleanupAttempts;
|
||||
|
||||
@Column(nullable = false, columnDefinition = "smallint")
|
||||
private int ttl;
|
||||
|
||||
|
||||
@ -3,7 +3,7 @@ package ru.serega6531.packmate.model.enums;
|
||||
public enum SubscriptionMessageType {
|
||||
SAVE_SERVICE, SAVE_PATTERN,
|
||||
DELETE_SERVICE,
|
||||
NEW_STREAM,
|
||||
NEW_STREAM, STREAMS_INVALIDATED,
|
||||
FINISH_LOOKBACK,
|
||||
COUNTERS_UPDATE,
|
||||
ENABLE_PATTERN, DISABLE_PATTERN,
|
||||
|
||||
@ -0,0 +1,19 @@
|
||||
package ru.serega6531.packmate.model.pojo;
|
||||
|
||||
public record PcapMetrics(
|
||||
long receivedPackets,
|
||||
long droppedPackets,
|
||||
long interfaceDroppedPackets,
|
||||
long applicationDroppedPackets,
|
||||
int queuedPackets,
|
||||
int maximumQueuedPackets,
|
||||
int activeProcessors,
|
||||
int processorCount,
|
||||
int queueCapacity,
|
||||
int captureBufferSizeBytes,
|
||||
String executorState,
|
||||
boolean captureOpen,
|
||||
boolean statsAvailable,
|
||||
long statsCapturedAt,
|
||||
String statsError
|
||||
) {}
|
||||
@ -13,12 +13,16 @@ public record UnfinishedStream(InetAddress firstIp, InetAddress secondIp, int fi
|
||||
return false;
|
||||
}
|
||||
|
||||
boolean ipEq1 = firstIp.equals(o.firstIp) && secondIp.equals(o.secondIp);
|
||||
boolean ipEq2 = firstIp.equals(o.secondIp) && secondIp.equals(o.firstIp);
|
||||
boolean portEq1 = firstPort == o.firstPort && secondPort == o.secondPort;
|
||||
boolean portEq2 = firstPort == o.secondPort && secondPort == o.firstPort;
|
||||
boolean sameDirection = firstIp.equals(o.firstIp)
|
||||
&& firstPort == o.firstPort
|
||||
&& secondIp.equals(o.secondIp)
|
||||
&& secondPort == o.secondPort;
|
||||
boolean reverseDirection = firstIp.equals(o.secondIp)
|
||||
&& firstPort == o.secondPort
|
||||
&& secondIp.equals(o.firstIp)
|
||||
&& secondPort == o.firstPort;
|
||||
|
||||
return (ipEq1 || ipEq2) && (portEq1 || portEq2) && protocol == o.protocol;
|
||||
return (sameDirection || reverseDirection) && protocol == o.protocol;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
package ru.serega6531.packmate.pcap;
|
||||
|
||||
import com.google.common.collect.*;
|
||||
import com.google.common.collect.ArrayListMultimap;
|
||||
import com.google.common.collect.HashMultimap;
|
||||
import com.google.common.collect.ListMultimap;
|
||||
import com.google.common.collect.Multimaps;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
|
||||
@ -9,7 +12,11 @@ import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.pcap4j.core.BpfProgram;
|
||||
import org.pcap4j.core.PacketListener;
|
||||
import org.pcap4j.core.PcapHandle;
|
||||
import org.pcap4j.packet.*;
|
||||
import org.pcap4j.packet.IpV4Packet;
|
||||
import org.pcap4j.packet.IpV6Packet;
|
||||
import org.pcap4j.packet.Packet;
|
||||
import org.pcap4j.packet.TcpPacket;
|
||||
import org.pcap4j.packet.UdpPacket;
|
||||
import ru.serega6531.packmate.model.CtfService;
|
||||
import ru.serega6531.packmate.model.enums.Protocol;
|
||||
import ru.serega6531.packmate.model.pojo.UnfinishedStream;
|
||||
@ -19,10 +26,21 @@ import ru.serega6531.packmate.service.StreamService;
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.FutureTask;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
@ -36,33 +54,69 @@ public abstract class AbstractPcapWorker implements PcapWorker, PacketListener {
|
||||
|
||||
protected String filter = "tcp or udp";
|
||||
|
||||
// во время работы должен быть != null
|
||||
protected ExecutorService processorExecutorService;
|
||||
|
||||
private final InetAddress localIp;
|
||||
|
||||
private long packetIdCounter = 0; // оно однопоточное, так что пусть будет без atomic
|
||||
|
||||
private final ListMultimap<UnfinishedStream, ru.serega6531.packmate.model.Packet> unfinishedTcpStreams = ArrayListMultimap.create();
|
||||
private final ListMultimap<UnfinishedStream, ru.serega6531.packmate.model.Packet> unfinishedUdpStreams = ArrayListMultimap.create();
|
||||
|
||||
// в следующих мапах в значениях находится srcIp соответствующего пакета
|
||||
private final SetMultimap<UnfinishedStream, Pair<InetAddress, Integer>> fins = HashMultimap.create();
|
||||
private final SetMultimap<UnfinishedStream, Pair<InetAddress, Integer>> acks = HashMultimap.create();
|
||||
private final AtomicLong packetIdCounter = new AtomicLong();
|
||||
private final Set<FutureTask<?>> controlTasks = ConcurrentHashMap.newKeySet();
|
||||
private volatile List<ProcessorShard> processorShards = List.of();
|
||||
|
||||
protected AbstractPcapWorker(ServicesService servicesService,
|
||||
StreamService streamService,
|
||||
InetAddress localIp) throws UnknownHostException {
|
||||
StreamService streamService,
|
||||
InetAddress localIp) throws UnknownHostException {
|
||||
this.servicesService = servicesService;
|
||||
this.streamService = streamService;
|
||||
|
||||
this.localIp = localIp;
|
||||
|
||||
BasicThreadFactory factory = new BasicThreadFactory.Builder()
|
||||
.namingPattern("pcap-loop").build();
|
||||
.namingPattern("pcap-loop-%d").build();
|
||||
loopExecutorService = Executors.newSingleThreadExecutor(factory);
|
||||
}
|
||||
|
||||
protected final synchronized void initializeProcessorShards(List<? extends ExecutorService> executors) {
|
||||
if (!processorShards.isEmpty()) {
|
||||
throw new IllegalStateException("Packet processor shards are already initialized");
|
||||
}
|
||||
if (executors.isEmpty()) {
|
||||
throw new IllegalArgumentException("At least one packet processor is required");
|
||||
}
|
||||
processorShards = executors.stream().map(ProcessorShard::new).toList();
|
||||
}
|
||||
|
||||
protected final List<ExecutorService> getProcessorExecutorServices() {
|
||||
return processorShards.stream().map(shard -> shard.executor).toList();
|
||||
}
|
||||
|
||||
protected final int getProcessorCount() {
|
||||
return processorShards.size();
|
||||
}
|
||||
|
||||
static int processorIndex(UnfinishedStream stream, int processorCount) {
|
||||
if (processorCount <= 0) {
|
||||
throw new IllegalArgumentException("processorCount must be positive");
|
||||
}
|
||||
byte[] firstAddress = stream.firstIp().getAddress();
|
||||
byte[] secondAddress = stream.secondIp().getAddress();
|
||||
int firstPort = stream.firstPort();
|
||||
int secondPort = stream.secondPort();
|
||||
int endpointOrder = Arrays.compareUnsigned(firstAddress, secondAddress);
|
||||
if (endpointOrder > 0 || (endpointOrder == 0 && firstPort > secondPort)) {
|
||||
byte[] addressSwap = firstAddress;
|
||||
firstAddress = secondAddress;
|
||||
secondAddress = addressSwap;
|
||||
int portSwap = firstPort;
|
||||
firstPort = secondPort;
|
||||
secondPort = portSwap;
|
||||
}
|
||||
|
||||
int hash = stream.protocol().hashCode();
|
||||
hash = 31 * hash + Arrays.hashCode(firstAddress);
|
||||
hash = 31 * hash + firstPort;
|
||||
hash = 31 * hash + Arrays.hashCode(secondAddress);
|
||||
hash = 31 * hash + secondPort;
|
||||
hash ^= hash >>> 16;
|
||||
return Math.floorMod(hash, processorCount);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void gotPacket(Packet rawPacket) {
|
||||
InetAddress sourceIp;
|
||||
InetAddress destIp;
|
||||
@ -85,11 +139,9 @@ public abstract class AbstractPcapWorker implements PcapWorker, PacketListener {
|
||||
final long time = pcap.getTimestamp().getTime();
|
||||
|
||||
if (rawPacket.contains(TcpPacket.class)) {
|
||||
final TcpPacket packet = rawPacket.get(TcpPacket.class);
|
||||
gotTcpPacket(packet, sourceIp, destIp, ttl, time);
|
||||
gotTcpPacket(rawPacket.get(TcpPacket.class), sourceIp, destIp, ttl, time);
|
||||
} else if (rawPacket.contains(UdpPacket.class)) {
|
||||
final UdpPacket packet = rawPacket.get(UdpPacket.class);
|
||||
gotUdpPacket(packet, sourceIp, destIp, ttl, time);
|
||||
gotUdpPacket(rawPacket.get(UdpPacket.class), sourceIp, destIp, ttl, time);
|
||||
}
|
||||
}
|
||||
|
||||
@ -104,26 +156,27 @@ public abstract class AbstractPcapWorker implements PcapWorker, PacketListener {
|
||||
|
||||
String sourceIpString = sourceIp.getHostAddress();
|
||||
String destIpString = destIp.getHostAddress();
|
||||
UnfinishedStream stream = new UnfinishedStream(sourceIp, destIp, sourcePort, destPort, Protocol.TCP);
|
||||
|
||||
final Optional<CtfService> serviceOptional =
|
||||
servicesService.findService(sourceIp, sourcePort, destIp, destPort);
|
||||
|
||||
if (serviceOptional.isPresent()) {
|
||||
processorExecutorService.execute(() -> {
|
||||
UnfinishedStream stream = addNewPacket(sourceIp, destIp, time, sourcePort, destPort, ttl, content, Protocol.TCP);
|
||||
dispatchPacket(stream, shard -> {
|
||||
addNewPacket(shard, stream, sourceIp, destIp, time, ttl, content);
|
||||
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("tcp {} {}:{} -> {}:{}, packet number {}",
|
||||
serviceOptional.get(), sourceIpString, sourcePort, destIpString, destPort,
|
||||
unfinishedTcpStreams.get(stream).size());
|
||||
shard.unfinishedTcpStreams.get(stream).size());
|
||||
}
|
||||
|
||||
checkTcpTermination(ack, fin, rst, new ImmutablePair<>(sourceIp, sourcePort), new ImmutablePair<>(destIp, destPort), stream);
|
||||
checkTcpTermination(shard, ack, fin, rst,
|
||||
new ImmutablePair<>(sourceIp, sourcePort),
|
||||
new ImmutablePair<>(destIp, destPort), stream);
|
||||
});
|
||||
} else { // сервис не найден
|
||||
if (log.isTraceEnabled()) {
|
||||
log.trace("tcp {}:{} -> {}:{}", sourceIpString, sourcePort, destIpString, destPort);
|
||||
}
|
||||
} else if (log.isTraceEnabled()) {
|
||||
log.trace("tcp {}:{} -> {}:{}", sourceIpString, sourcePort, destIpString, destPort);
|
||||
}
|
||||
}
|
||||
|
||||
@ -135,34 +188,64 @@ public abstract class AbstractPcapWorker implements PcapWorker, PacketListener {
|
||||
|
||||
String sourceIpString = sourceIp.getHostAddress();
|
||||
String destIpString = destIp.getHostAddress();
|
||||
UnfinishedStream stream = new UnfinishedStream(sourceIp, destIp, sourcePort, destPort, Protocol.UDP);
|
||||
|
||||
final Optional<CtfService> serviceOptional =
|
||||
servicesService.findService(sourceIp, sourcePort, destIp, destPort);
|
||||
|
||||
if (serviceOptional.isPresent()) {
|
||||
processorExecutorService.execute(() -> {
|
||||
UnfinishedStream stream = addNewPacket(sourceIp, destIp, time, sourcePort, destPort, ttl, content, Protocol.UDP);
|
||||
dispatchPacket(stream, shard -> {
|
||||
addNewPacket(shard, stream, sourceIp, destIp, time, ttl, content);
|
||||
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("udp {} {}:{} -> {}:{}, packet number {}",
|
||||
serviceOptional.get(), sourceIpString, sourcePort, destIpString, destPort,
|
||||
unfinishedUdpStreams.get(stream).size());
|
||||
shard.unfinishedUdpStreams.get(stream).size());
|
||||
}
|
||||
});
|
||||
} else { // сервис не найден
|
||||
if (log.isTraceEnabled()) {
|
||||
log.trace("udp {}:{} -> {}:{}", sourceIpString, sourcePort, destIpString, destPort);
|
||||
}
|
||||
} else if (log.isTraceEnabled()) {
|
||||
log.trace("udp {}:{} -> {}:{}", sourceIpString, sourcePort, destIpString, destPort);
|
||||
}
|
||||
}
|
||||
|
||||
private UnfinishedStream addNewPacket(InetAddress sourceIp, InetAddress destIp, long time,
|
||||
int sourcePort, int destPort, int ttl, byte[] content, Protocol protocol) {
|
||||
var incoming = destIp.equals(localIp);
|
||||
var stream = new UnfinishedStream(sourceIp, destIp, sourcePort, destPort, protocol);
|
||||
private void dispatchPacket(UnfinishedStream stream, ProcessorAction action) {
|
||||
List<ProcessorShard> shards = processorShards;
|
||||
if (shards.isEmpty()) {
|
||||
throw new IllegalStateException("Packet processor shards are not initialized");
|
||||
}
|
||||
ProcessorShard shard = shards.get(processorIndex(stream, shards.size()));
|
||||
try {
|
||||
onPacketQueued();
|
||||
shard.executor.execute(() -> {
|
||||
onPacketDequeued();
|
||||
action.run(shard);
|
||||
});
|
||||
} catch (RejectedExecutionException exception) {
|
||||
onPacketDequeued();
|
||||
onPacketRejected(stream, exception);
|
||||
}
|
||||
}
|
||||
|
||||
protected void onPacketQueued() {
|
||||
}
|
||||
|
||||
protected void onPacketDequeued() {
|
||||
}
|
||||
|
||||
protected void onPacketRejected(UnfinishedStream stream, RejectedExecutionException exception) {
|
||||
throw exception;
|
||||
}
|
||||
|
||||
private void addNewPacket(ProcessorShard shard,
|
||||
UnfinishedStream stream,
|
||||
InetAddress sourceIp,
|
||||
InetAddress destIp,
|
||||
long time,
|
||||
int ttl,
|
||||
byte[] content) {
|
||||
var incoming = destIp.equals(localIp);
|
||||
var packet = ru.serega6531.packmate.model.Packet.builder()
|
||||
.tempId(packetIdCounter++)
|
||||
.tempId(packetIdCounter.getAndIncrement())
|
||||
.ttl(ttl)
|
||||
.timestamp(time)
|
||||
.sourceIp(sourceIp.getHostAddress())
|
||||
@ -171,91 +254,218 @@ public abstract class AbstractPcapWorker implements PcapWorker, PacketListener {
|
||||
.content(content)
|
||||
.build();
|
||||
|
||||
final var streams = (protocol == Protocol.TCP) ? this.unfinishedTcpStreams : this.unfinishedUdpStreams;
|
||||
final var streams = stream.protocol() == Protocol.TCP
|
||||
? shard.unfinishedTcpStreams
|
||||
: shard.unfinishedUdpStreams;
|
||||
|
||||
if (!streams.containsKey(stream)) {
|
||||
log.debug("New stream started");
|
||||
}
|
||||
|
||||
streams.put(stream, packet);
|
||||
return stream;
|
||||
}
|
||||
|
||||
/**
|
||||
* Udp не имеет фазы закрытия, поэтому закрывается только по таймауту
|
||||
*/
|
||||
private void checkTcpTermination(boolean ack, boolean fin, boolean rst,
|
||||
private void checkTcpTermination(ProcessorShard shard,
|
||||
boolean ack,
|
||||
boolean fin,
|
||||
boolean rst,
|
||||
Pair<InetAddress, Integer> sourceIpAndPort,
|
||||
Pair<InetAddress, Integer> destIpAndPort,
|
||||
UnfinishedStream stream) {
|
||||
|
||||
if (fin) {
|
||||
fins.put(stream, sourceIpAndPort);
|
||||
shard.fins.put(stream, sourceIpAndPort);
|
||||
}
|
||||
|
||||
if (ack && fins.containsEntry(stream, destIpAndPort)) { // проверяем destIp, потому что ищем ответ на его fin
|
||||
acks.put(stream, sourceIpAndPort);
|
||||
if (ack && shard.fins.containsEntry(stream, destIpAndPort)) {
|
||||
shard.acks.put(stream, sourceIpAndPort);
|
||||
}
|
||||
|
||||
// если соединение разорвано с помощью rst или закрыто с помощью fin-ack-fin-ack
|
||||
if (rst || (acks.containsEntry(stream, sourceIpAndPort) && acks.containsEntry(stream, destIpAndPort))) {
|
||||
streamService.saveNewStream(stream, unfinishedTcpStreams.get(stream));
|
||||
|
||||
unfinishedTcpStreams.removeAll(stream);
|
||||
fins.removeAll(stream);
|
||||
acks.removeAll(stream);
|
||||
if (rst || (shard.acks.containsEntry(stream, sourceIpAndPort)
|
||||
&& shard.acks.containsEntry(stream, destIpAndPort))) {
|
||||
streamService.saveNewStream(stream, shard.unfinishedTcpStreams.get(stream));
|
||||
shard.unfinishedTcpStreams.removeAll(stream);
|
||||
shard.fins.removeAll(stream);
|
||||
shard.acks.removeAll(stream);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public void closeAllStreams(Protocol protocol) {
|
||||
final var streams = (protocol == Protocol.TCP) ? this.unfinishedTcpStreams : this.unfinishedUdpStreams;
|
||||
List<Future<Integer>> futures = new ArrayList<>(processorShards.size());
|
||||
for (ProcessorShard shard : processorShards) {
|
||||
futures.add(submitControl(shard, () -> closeAllStreams(shard, protocol)));
|
||||
}
|
||||
for (Future<Integer> future : futures) {
|
||||
future.get();
|
||||
}
|
||||
}
|
||||
|
||||
private int closeAllStreams(ProcessorShard shard, Protocol protocol) {
|
||||
final var streams = protocol == Protocol.TCP
|
||||
? shard.unfinishedTcpStreams
|
||||
: shard.unfinishedUdpStreams;
|
||||
int streamsClosed = streams.keySet().size();
|
||||
|
||||
Multimaps.asMap(streams).forEach((key, value) ->
|
||||
streamService.saveNewStream(key, new ArrayList<>(value)));
|
||||
|
||||
streams.clear();
|
||||
if (protocol == Protocol.TCP) {
|
||||
fins.clear();
|
||||
acks.clear();
|
||||
shard.fins.clear();
|
||||
shard.acks.clear();
|
||||
}
|
||||
return streamsClosed;
|
||||
}
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public int closeTimeoutStreams(Protocol protocol, long timeoutMillis) {
|
||||
return processorExecutorService.submit(() -> {
|
||||
int streamsClosed = 0;
|
||||
List<Future<Integer>> futures = new ArrayList<>(processorShards.size());
|
||||
for (ProcessorShard shard : processorShards) {
|
||||
futures.add(submitControl(shard, () -> closeTimeoutStreams(shard, protocol, timeoutMillis)));
|
||||
}
|
||||
|
||||
final long time = System.currentTimeMillis();
|
||||
final var streams = (protocol == Protocol.TCP) ? this.unfinishedTcpStreams : this.unfinishedUdpStreams;
|
||||
int streamsClosed = 0;
|
||||
for (Future<Integer> future : futures) {
|
||||
streamsClosed += future.get();
|
||||
}
|
||||
return streamsClosed;
|
||||
}
|
||||
|
||||
final var oldStreams = Multimaps.asMap(streams).entrySet().stream()
|
||||
.filter(entry -> {
|
||||
final var packets = entry.getValue();
|
||||
return time - packets.get(packets.size() - 1).getTimestamp() > timeoutMillis;
|
||||
})
|
||||
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
|
||||
private int closeTimeoutStreams(ProcessorShard shard, Protocol protocol, long timeoutMillis) {
|
||||
int streamsClosed = 0;
|
||||
final long time = System.currentTimeMillis();
|
||||
final var streams = protocol == Protocol.TCP
|
||||
? shard.unfinishedTcpStreams
|
||||
: shard.unfinishedUdpStreams;
|
||||
|
||||
for (var entry : oldStreams.entrySet()) {
|
||||
final UnfinishedStream stream = entry.getKey();
|
||||
final var packets = entry.getValue();
|
||||
final var oldStreams = Multimaps.asMap(streams).entrySet().stream()
|
||||
.filter(entry -> {
|
||||
final var packets = entry.getValue();
|
||||
return time - packets.get(packets.size() - 1).getTimestamp() > timeoutMillis;
|
||||
})
|
||||
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
|
||||
|
||||
if (streamService.saveNewStream(stream, packets)) {
|
||||
streamsClosed++;
|
||||
}
|
||||
for (var entry : oldStreams.entrySet()) {
|
||||
final UnfinishedStream stream = entry.getKey();
|
||||
final var packets = entry.getValue();
|
||||
|
||||
if (protocol == Protocol.TCP) {
|
||||
fins.removeAll(stream);
|
||||
acks.removeAll(stream);
|
||||
}
|
||||
|
||||
streams.removeAll(stream);
|
||||
if (streamService.saveNewStream(stream, packets)) {
|
||||
streamsClosed++;
|
||||
}
|
||||
|
||||
return streamsClosed;
|
||||
}).get();
|
||||
if (protocol == Protocol.TCP) {
|
||||
shard.fins.removeAll(stream);
|
||||
shard.acks.removeAll(stream);
|
||||
}
|
||||
streams.removeAll(stream);
|
||||
}
|
||||
return streamsClosed;
|
||||
}
|
||||
|
||||
private <T> Future<T> submitControl(ProcessorShard shard, Callable<T> callable) throws InterruptedException {
|
||||
return submitControl(shard, callable, System.nanoTime() + TimeUnit.SECONDS.toNanos(5));
|
||||
}
|
||||
|
||||
private <T> Future<T> submitControl(ProcessorShard shard,
|
||||
Callable<T> callable,
|
||||
long enqueueDeadlineNanos) throws InterruptedException {
|
||||
FutureTask<T> task = new FutureTask<>(callable) {
|
||||
@Override
|
||||
protected void done() {
|
||||
controlTasks.remove(this);
|
||||
}
|
||||
};
|
||||
controlTasks.add(task);
|
||||
if (!(shard.executor instanceof ThreadPoolExecutor executor)) {
|
||||
try {
|
||||
shard.executor.execute(task);
|
||||
return task;
|
||||
} catch (RejectedExecutionException exception) {
|
||||
task.cancel(false);
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
if (executor.isShutdown()) {
|
||||
task.cancel(false);
|
||||
throw new RejectedExecutionException("Packet processor is shutting down");
|
||||
}
|
||||
|
||||
try {
|
||||
executor.execute(task);
|
||||
} catch (RejectedExecutionException exception) {
|
||||
if (executor.isShutdown()) {
|
||||
task.cancel(false);
|
||||
throw exception;
|
||||
}
|
||||
long remaining = enqueueDeadlineNanos - System.nanoTime();
|
||||
boolean enqueued;
|
||||
try {
|
||||
enqueued = remaining > 0
|
||||
&& executor.getQueue().offer(task, remaining, TimeUnit.NANOSECONDS);
|
||||
} catch (InterruptedException interruptedException) {
|
||||
task.cancel(false);
|
||||
throw interruptedException;
|
||||
}
|
||||
if (!enqueued) {
|
||||
task.cancel(false);
|
||||
throw new RejectedExecutionException(
|
||||
"Timed out enqueueing packet processor control task", exception);
|
||||
}
|
||||
if (executor.isShutdown() && executor.getQueue().remove(task)) {
|
||||
task.cancel(false);
|
||||
throw new RejectedExecutionException("Packet processor shut down while enqueueing control task");
|
||||
}
|
||||
}
|
||||
return task;
|
||||
}
|
||||
|
||||
protected final boolean flushAndShutdownProcessors(int timeoutSeconds) {
|
||||
long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(Math.max(1, timeoutSeconds));
|
||||
List<Future<Integer>> flushes = new ArrayList<>(processorShards.size());
|
||||
try {
|
||||
for (ProcessorShard shard : processorShards) {
|
||||
flushes.add(submitControl(
|
||||
shard,
|
||||
() -> closeAllStreams(shard, Protocol.TCP) + closeAllStreams(shard, Protocol.UDP),
|
||||
deadline
|
||||
));
|
||||
}
|
||||
processorShards.forEach(this::shutdownProcessor);
|
||||
for (Future<Integer> flush : flushes) {
|
||||
long remaining = deadline - System.nanoTime();
|
||||
if (remaining <= 0) {
|
||||
throw new java.util.concurrent.TimeoutException("Packet processor shutdown timed out");
|
||||
}
|
||||
flush.get(remaining, TimeUnit.NANOSECONDS);
|
||||
}
|
||||
for (ProcessorShard shard : processorShards) {
|
||||
long remaining = deadline - System.nanoTime();
|
||||
if (remaining <= 0 || !shard.executor.awaitTermination(remaining, TimeUnit.NANOSECONDS)) {
|
||||
throw new java.util.concurrent.TimeoutException("Packet processor shutdown timed out");
|
||||
}
|
||||
}
|
||||
return true;
|
||||
} catch (Exception exception) {
|
||||
log.error("Packet processors did not stop cleanly", exception);
|
||||
controlTasks.forEach(task -> task.cancel(true));
|
||||
processorShards.forEach(this::shutdownNowAndCancelQueuedTasks);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private void shutdownProcessor(ProcessorShard shard) {
|
||||
shard.executor.shutdown();
|
||||
}
|
||||
|
||||
private void shutdownNowAndCancelQueuedTasks(ProcessorShard shard) {
|
||||
List<Runnable> queuedTasks = shard.executor.shutdownNow();
|
||||
queuedTasks.forEach(task -> {
|
||||
if (task instanceof Future<?> future) {
|
||||
future.cancel(false);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -266,8 +476,29 @@ public abstract class AbstractPcapWorker implements PcapWorker, PacketListener {
|
||||
|
||||
@SneakyThrows
|
||||
protected void applyFilter() {
|
||||
if(filter != null && pcap != null && pcap.isOpen()) {
|
||||
if (filter != null && pcap != null && pcap.isOpen()) {
|
||||
pcap.setFilter(filter, BpfProgram.BpfCompileMode.OPTIMIZE);
|
||||
}
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
private interface ProcessorAction {
|
||||
void run(ProcessorShard shard);
|
||||
}
|
||||
|
||||
private static final class ProcessorShard {
|
||||
private final ExecutorService executor;
|
||||
private final ListMultimap<UnfinishedStream, ru.serega6531.packmate.model.Packet> unfinishedTcpStreams =
|
||||
ArrayListMultimap.create();
|
||||
private final ListMultimap<UnfinishedStream, ru.serega6531.packmate.model.Packet> unfinishedUdpStreams =
|
||||
ArrayListMultimap.create();
|
||||
private final com.google.common.collect.SetMultimap<UnfinishedStream, Pair<InetAddress, Integer>> fins =
|
||||
HashMultimap.create();
|
||||
private final com.google.common.collect.SetMultimap<UnfinishedStream, Pair<InetAddress, Integer>> acks =
|
||||
HashMultimap.create();
|
||||
|
||||
private ProcessorShard(ExecutorService executor) {
|
||||
this.executor = executor;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -10,6 +10,7 @@ import ru.serega6531.packmate.exception.PcapFileNotFoundException;
|
||||
import ru.serega6531.packmate.model.enums.Protocol;
|
||||
import ru.serega6531.packmate.model.enums.SubscriptionMessageType;
|
||||
import ru.serega6531.packmate.model.pojo.SubscriptionMessage;
|
||||
import ru.serega6531.packmate.model.pojo.PcapMetrics;
|
||||
import ru.serega6531.packmate.service.ServicesService;
|
||||
import ru.serega6531.packmate.service.StreamService;
|
||||
import ru.serega6531.packmate.service.SubscriptionService;
|
||||
@ -18,6 +19,7 @@ import java.io.EOFException;
|
||||
import java.io.File;
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.List;
|
||||
|
||||
@Slf4j
|
||||
public class FilePcapWorker extends AbstractPcapWorker {
|
||||
@ -38,7 +40,7 @@ public class FilePcapWorker extends AbstractPcapWorker {
|
||||
file = new File(directory, filename);
|
||||
validateFileExists();
|
||||
|
||||
processorExecutorService = new InlineExecutorService();
|
||||
initializeProcessorShards(List.of(new InlineExecutorService()));
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
@ -87,6 +89,13 @@ public class FilePcapWorker extends AbstractPcapWorker {
|
||||
return "inline";
|
||||
}
|
||||
|
||||
@Override
|
||||
public PcapMetrics getMetrics() {
|
||||
return new PcapMetrics(0, 0, 0, 0, 0, 0, 0, 1, 0, 0,
|
||||
"inline", false, false, 0,
|
||||
"pcap statistics are unavailable for file capture");
|
||||
}
|
||||
|
||||
private void validateFileExists() {
|
||||
if (!file.exists()) {
|
||||
throw new PcapFileNotFoundException(file, directory);
|
||||
|
||||
@ -3,45 +3,120 @@ package ru.serega6531.packmate.pcap;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
|
||||
import org.pcap4j.core.NotOpenException;
|
||||
import org.pcap4j.core.PcapNativeException;
|
||||
import org.pcap4j.core.PcapNetworkInterface;
|
||||
import org.pcap4j.core.PcapStat;
|
||||
import org.pcap4j.core.Pcaps;
|
||||
import ru.serega6531.packmate.exception.PcapInterfaceNotFoundException;
|
||||
import ru.serega6531.packmate.model.pojo.PcapMetrics;
|
||||
import ru.serega6531.packmate.model.pojo.UnfinishedStream;
|
||||
import ru.serega6531.packmate.properties.PackmateProperties;
|
||||
import ru.serega6531.packmate.service.ServicesService;
|
||||
import ru.serega6531.packmate.service.StreamService;
|
||||
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
public class LivePcapWorker extends AbstractPcapWorker {
|
||||
|
||||
private static final int MAX_PROCESSOR_WORKERS = 64;
|
||||
|
||||
private final PcapNetworkInterface device;
|
||||
private final List<ThreadPoolExecutor> processorExecutors;
|
||||
private final int totalQueueCapacity;
|
||||
private final int bufferSizeBytes;
|
||||
private final int shutdownTimeoutSeconds;
|
||||
private final AtomicBoolean stopped = new AtomicBoolean();
|
||||
private final AtomicLong applicationDroppedPackets = new AtomicLong();
|
||||
private final AtomicInteger queuedPackets = new AtomicInteger();
|
||||
private final AtomicInteger maximumQueuedPackets = new AtomicInteger();
|
||||
|
||||
private volatile long lastReceived;
|
||||
private volatile long lastDropped;
|
||||
private volatile long lastInterfaceDropped;
|
||||
private volatile long lastStatsCapturedAt;
|
||||
private volatile String lastStatsError = "pcap statistics have not been read yet";
|
||||
|
||||
public LivePcapWorker(ServicesService servicesService,
|
||||
StreamService streamService,
|
||||
InetAddress localIp,
|
||||
String interfaceName) throws PcapNativeException, UnknownHostException {
|
||||
String interfaceName,
|
||||
PackmateProperties.CaptureProperties captureProperties)
|
||||
throws PcapNativeException, UnknownHostException {
|
||||
super(servicesService, streamService, localIp);
|
||||
device = Pcaps.getDevByName(interfaceName);
|
||||
|
||||
if (device == null) {
|
||||
List<String> existingInterfaces = Pcaps.findAllDevs().stream().map(PcapNetworkInterface::getName).toList();
|
||||
List<String> existingInterfaces = Pcaps.findAllDevs().stream()
|
||||
.map(PcapNetworkInterface::getName)
|
||||
.toList();
|
||||
throw new PcapInterfaceNotFoundException(interfaceName, existingInterfaces);
|
||||
}
|
||||
|
||||
BasicThreadFactory factory = new BasicThreadFactory.Builder()
|
||||
.namingPattern("pcap-processor").build();
|
||||
processorExecutorService = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), factory);
|
||||
int workerCount = captureProperties.workers();
|
||||
if (workerCount < 1 || workerCount > MAX_PROCESSOR_WORKERS) {
|
||||
throw new IllegalArgumentException("packmate.capture.workers must be between 1 and "
|
||||
+ MAX_PROCESSOR_WORKERS);
|
||||
}
|
||||
if (captureProperties.queueCapacity() < workerCount) {
|
||||
throw new IllegalArgumentException("packmate.capture.queue-capacity must be at least the worker count");
|
||||
}
|
||||
if (captureProperties.shutdownTimeoutSeconds() < 1) {
|
||||
throw new IllegalArgumentException("packmate.capture.shutdown-timeout-seconds must be positive");
|
||||
}
|
||||
if (captureProperties.bufferSizeBytes() < 1_048_576) {
|
||||
throw new IllegalArgumentException("packmate.capture.buffer-size-bytes must be at least 1048576");
|
||||
}
|
||||
|
||||
int queueCapacityPerWorker = Math.floorDiv(
|
||||
captureProperties.queueCapacity() + workerCount - 1,
|
||||
workerCount
|
||||
);
|
||||
totalQueueCapacity = Math.multiplyExact(queueCapacityPerWorker, workerCount);
|
||||
bufferSizeBytes = captureProperties.bufferSizeBytes();
|
||||
shutdownTimeoutSeconds = captureProperties.shutdownTimeoutSeconds();
|
||||
processorExecutors = new ArrayList<>(workerCount);
|
||||
|
||||
for (int index = 0; index < workerCount; index++) {
|
||||
BasicThreadFactory factory = new BasicThreadFactory.Builder()
|
||||
.namingPattern("pcap-processor-" + index + "-%d")
|
||||
.build();
|
||||
ThreadPoolExecutor executor = new ThreadPoolExecutor(
|
||||
1,
|
||||
1,
|
||||
0L,
|
||||
TimeUnit.MILLISECONDS,
|
||||
new ArrayBlockingQueue<>(queueCapacityPerWorker),
|
||||
factory,
|
||||
new ThreadPoolExecutor.AbortPolicy()
|
||||
);
|
||||
processorExecutors.add(executor);
|
||||
}
|
||||
initializeProcessorShards(new ArrayList<ExecutorService>(processorExecutors));
|
||||
}
|
||||
|
||||
public void start() throws PcapNativeException {
|
||||
log.info("Using interface " + device.getName());
|
||||
pcap = device.openLive(65536, PcapNetworkInterface.PromiscuousMode.PROMISCUOUS, 100);
|
||||
log.info("Using interface {} with {} packet processors, total queue capacity {}, capture buffer {} bytes",
|
||||
device.getName(), getProcessorCount(), totalQueueCapacity, bufferSizeBytes);
|
||||
pcap = new org.pcap4j.core.PcapHandle.Builder(device.getName())
|
||||
.snaplen(65536)
|
||||
.promiscuousMode(PcapNetworkInterface.PromiscuousMode.PROMISCUOUS)
|
||||
.timeoutMillis(100)
|
||||
.bufferSize(bufferSizeBytes)
|
||||
.build();
|
||||
|
||||
applyFilter();
|
||||
|
||||
@ -51,26 +126,97 @@ public class LivePcapWorker extends AbstractPcapWorker {
|
||||
pcap.loop(-1, this);
|
||||
} catch (InterruptedException ignored) {
|
||||
Thread.currentThread().interrupt();
|
||||
// выходим
|
||||
} catch (Exception e) {
|
||||
log.error("Error while capturing packet", e);
|
||||
} catch (Exception exception) {
|
||||
log.error("Error while capturing packet", exception);
|
||||
stop();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onPacketQueued() {
|
||||
int queued = queuedPackets.incrementAndGet();
|
||||
maximumQueuedPackets.accumulateAndGet(queued, Math::max);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onPacketDequeued() {
|
||||
queuedPackets.decrementAndGet();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onPacketRejected(UnfinishedStream stream, RejectedExecutionException exception) {
|
||||
long dropped = applicationDroppedPackets.incrementAndGet();
|
||||
if (dropped == 1 || dropped % 10_000 == 0) {
|
||||
log.error("Packet processor queue is full; {} application packets dropped so far", dropped);
|
||||
}
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
public void stop() {
|
||||
if (!stopped.compareAndSet(false, true)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (pcap != null && pcap.isOpen()) {
|
||||
pcap.breakLoop();
|
||||
pcap.close();
|
||||
}
|
||||
loopExecutorService.shutdown();
|
||||
|
||||
log.info("Intercept stopped");
|
||||
boolean drained = flushAndShutdownProcessors(shutdownTimeoutSeconds);
|
||||
if (drained) {
|
||||
log.info("Intercept stopped after draining packet processors");
|
||||
} else {
|
||||
log.error("Intercept stopped before all queued packets could be drained");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getExecutorState() {
|
||||
return processorExecutorService.toString();
|
||||
return processorExecutors.stream()
|
||||
.map(ThreadPoolExecutor::toString)
|
||||
.collect(Collectors.joining(", ", "[", "]"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public PcapMetrics getMetrics() {
|
||||
boolean captureOpen = pcap != null && pcap.isOpen();
|
||||
boolean statsAvailable = false;
|
||||
if (captureOpen) {
|
||||
try {
|
||||
PcapStat stat = pcap.getStats();
|
||||
lastReceived = stat.getNumPacketsReceived();
|
||||
lastDropped = stat.getNumPacketsDropped();
|
||||
lastInterfaceDropped = stat.getNumPacketsDroppedByIf();
|
||||
lastStatsCapturedAt = System.currentTimeMillis();
|
||||
lastStatsError = null;
|
||||
statsAvailable = true;
|
||||
} catch (PcapNativeException | NotOpenException exception) {
|
||||
lastStatsError = exception.getClass().getSimpleName() + ": " + exception.getMessage();
|
||||
log.warn("Failed to read pcap statistics", exception);
|
||||
}
|
||||
} else {
|
||||
lastStatsError = "live capture handle is closed";
|
||||
}
|
||||
|
||||
int activeProcessors = processorExecutors.stream().mapToInt(ThreadPoolExecutor::getActiveCount).sum();
|
||||
return new PcapMetrics(
|
||||
lastReceived,
|
||||
lastDropped,
|
||||
lastInterfaceDropped,
|
||||
applicationDroppedPackets.get(),
|
||||
queuedPackets.get(),
|
||||
maximumQueuedPackets.get(),
|
||||
activeProcessors,
|
||||
getProcessorCount(),
|
||||
totalQueueCapacity,
|
||||
bufferSizeBytes,
|
||||
getExecutorState(),
|
||||
captureOpen,
|
||||
statsAvailable,
|
||||
lastStatsCapturedAt,
|
||||
lastStatsError
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package ru.serega6531.packmate.pcap;
|
||||
|
||||
import ru.serega6531.packmate.model.enums.Protocol;
|
||||
import ru.serega6531.packmate.model.pojo.PcapMetrics;
|
||||
|
||||
public class NoOpPcapWorker implements PcapWorker {
|
||||
@Override
|
||||
@ -28,4 +29,11 @@ public class NoOpPcapWorker implements PcapWorker {
|
||||
public String getExecutorState() {
|
||||
return "none";
|
||||
}
|
||||
|
||||
@Override
|
||||
public PcapMetrics getMetrics() {
|
||||
return new PcapMetrics(0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
"none", false, false, 0,
|
||||
"packet capture is disabled");
|
||||
}
|
||||
}
|
||||
|
||||
@ -2,6 +2,7 @@ package ru.serega6531.packmate.pcap;
|
||||
|
||||
import org.pcap4j.core.PcapNativeException;
|
||||
import ru.serega6531.packmate.model.enums.Protocol;
|
||||
import ru.serega6531.packmate.model.pojo.PcapMetrics;
|
||||
|
||||
public interface PcapWorker {
|
||||
|
||||
@ -21,4 +22,6 @@ public interface PcapWorker {
|
||||
void setFilter(String filter);
|
||||
|
||||
String getExecutorState();
|
||||
|
||||
PcapMetrics getMetrics();
|
||||
}
|
||||
|
||||
@ -14,6 +14,7 @@ public record PackmateProperties(
|
||||
InetAddress localIp,
|
||||
WebProperties web,
|
||||
TimeoutProperties timeout,
|
||||
CaptureProperties capture,
|
||||
CleanupProperties cleanup,
|
||||
boolean ignoreEmptyPackets
|
||||
) {
|
||||
@ -29,10 +30,26 @@ public record PackmateProperties(
|
||||
int checkInterval
|
||||
){}
|
||||
|
||||
public record CaptureProperties(
|
||||
int workers,
|
||||
int queueCapacity,
|
||||
int bufferSizeBytes,
|
||||
int shutdownTimeoutSeconds
|
||||
){}
|
||||
|
||||
public record CleanupProperties(
|
||||
boolean enabled,
|
||||
int threshold,
|
||||
int interval
|
||||
int intervalSeconds,
|
||||
int maxRunSeconds,
|
||||
int commitGraceSeconds,
|
||||
int batchSize,
|
||||
int maxBatchesPerRun,
|
||||
int maxPacketsPerBatch,
|
||||
int maxMatchesPerBatch,
|
||||
long maxBytesPerBatch,
|
||||
int statementTimeoutSeconds,
|
||||
int lockTimeoutSeconds
|
||||
){}
|
||||
|
||||
}
|
||||
|
||||
@ -5,6 +5,7 @@ import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
|
||||
import org.springframework.data.jpa.repository.Modifying;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
import org.springframework.data.repository.query.Param;
|
||||
import ru.serega6531.packmate.model.Packet;
|
||||
import ru.serega6531.packmate.model.Stream;
|
||||
|
||||
@ -12,18 +13,146 @@ import java.util.List;
|
||||
|
||||
public interface StreamRepository extends JpaRepository<Stream, Long>, JpaSpecificationExecutor<Stream> {
|
||||
|
||||
@Query("UPDATE Stream SET favorite = :favorite WHERE id = :id")
|
||||
@Query("UPDATE Stream SET favorite = :favorite WHERE id = :id AND cleanupPending = false")
|
||||
@Modifying
|
||||
void setFavorite(long id, boolean favorite);
|
||||
|
||||
long deleteByEndTimestampBeforeAndFavoriteIsFalse(long threshold);
|
||||
@Query("SELECT s.id FROM Stream s WHERE s.id IN :ids AND s.cleanupPending = false")
|
||||
List<Long> findAvailableStreamIds(@Param("ids") List<Long> ids);
|
||||
|
||||
@Query(value = """
|
||||
SELECT id,
|
||||
packets_count AS "packetsCount",
|
||||
size_bytes AS "sizeBytes",
|
||||
cleanup_attempts AS "attempts"
|
||||
FROM stream
|
||||
WHERE end_timestamp < :threshold
|
||||
AND persisted_at < :persistedThreshold
|
||||
AND favorite = false
|
||||
AND cleanup_pending = false
|
||||
AND cleanup_retry_after <= :retryNow
|
||||
ORDER BY end_timestamp, id
|
||||
LIMIT :batchSize
|
||||
FOR UPDATE SKIP LOCKED
|
||||
""", nativeQuery = true)
|
||||
List<CleanupCandidate> findAndLockOldStreamCandidates(@Param("threshold") long threshold,
|
||||
@Param("persistedThreshold") long persistedThreshold,
|
||||
@Param("retryNow") long retryNow,
|
||||
@Param("batchSize") int batchSize);
|
||||
|
||||
@Query(value = """
|
||||
SELECT id,
|
||||
packets_count AS "packetsCount",
|
||||
size_bytes AS "sizeBytes",
|
||||
cleanup_attempts AS "attempts"
|
||||
FROM stream
|
||||
WHERE cleanup_pending = true
|
||||
AND cleanup_retry_after <= :retryNow
|
||||
ORDER BY end_timestamp, id
|
||||
LIMIT :batchSize
|
||||
FOR UPDATE SKIP LOCKED
|
||||
""", nativeQuery = true)
|
||||
List<CleanupCandidate> findAndLockPendingStreams(@Param("retryNow") long retryNow,
|
||||
@Param("batchSize") int batchSize);
|
||||
|
||||
@Modifying
|
||||
@Query(value = "UPDATE stream SET cleanup_pending = true WHERE id IN (:streamIds)", nativeQuery = true)
|
||||
int markCleanupPending(@Param("streamIds") List<Long> streamIds);
|
||||
|
||||
@Modifying
|
||||
@Query(value = """
|
||||
UPDATE stream
|
||||
SET cleanup_attempts = cleanup_attempts + 1,
|
||||
cleanup_retry_after = :retryAfter
|
||||
WHERE id IN (:streamIds)
|
||||
""", nativeQuery = true)
|
||||
int deferCleanup(@Param("streamIds") List<Long> streamIds,
|
||||
@Param("retryAfter") long retryAfter);
|
||||
|
||||
@Modifying
|
||||
@Query(value = """
|
||||
UPDATE stream
|
||||
SET cleanup_attempts = 0,
|
||||
cleanup_retry_after = 0
|
||||
WHERE id IN (:streamIds)
|
||||
""", nativeQuery = true)
|
||||
int resetCleanupRetry(@Param("streamIds") List<Long> streamIds);
|
||||
|
||||
@Query(value = """
|
||||
SELECT set_config('statement_timeout', CAST(:statementTimeoutMillis AS text), true)
|
||||
|| set_config('lock_timeout', CAST(:lockTimeoutMillis AS text), true)
|
||||
""", nativeQuery = true)
|
||||
String configureLocalTimeouts(@Param("statementTimeoutMillis") long statementTimeoutMillis,
|
||||
@Param("lockTimeoutMillis") long lockTimeoutMillis);
|
||||
|
||||
@Modifying
|
||||
@Query(value = "DELETE FROM stream_found_patterns WHERE stream_id IN (:streamIds)", nativeQuery = true)
|
||||
int deletePatternLinks(@Param("streamIds") List<Long> streamIds);
|
||||
|
||||
@Query(value = """
|
||||
SELECT id, octet_length(content) AS "sizeBytes"
|
||||
FROM packet
|
||||
WHERE stream_id IN (:streamIds)
|
||||
ORDER BY stream_id, id
|
||||
LIMIT :maxPackets
|
||||
""", nativeQuery = true)
|
||||
List<CleanupPacket> findPacketsForCleanup(@Param("streamIds") List<Long> streamIds,
|
||||
@Param("maxPackets") int maxPackets);
|
||||
|
||||
@Query(value = """
|
||||
SELECT id
|
||||
FROM found_pattern
|
||||
WHERE packet_id IN (:packetIds)
|
||||
ORDER BY id
|
||||
LIMIT :maxMatches
|
||||
""", nativeQuery = true)
|
||||
List<Long> findFoundPatternsForCleanup(@Param("packetIds") List<Long> packetIds,
|
||||
@Param("maxMatches") int maxMatches);
|
||||
|
||||
@Modifying
|
||||
@Query(value = "DELETE FROM found_pattern WHERE id IN (:matchIds)", nativeQuery = true)
|
||||
int deleteFoundPatterns(@Param("matchIds") List<Long> matchIds);
|
||||
|
||||
@Modifying
|
||||
@Query(value = """
|
||||
DELETE FROM packet p
|
||||
WHERE p.id IN (:packetIds)
|
||||
AND NOT EXISTS (SELECT 1 FROM found_pattern fp WHERE fp.packet_id = p.id)
|
||||
""", nativeQuery = true)
|
||||
int deletePacketsWithoutMatches(@Param("packetIds") List<Long> packetIds);
|
||||
|
||||
@Modifying
|
||||
@Query(value = """
|
||||
DELETE FROM stream s
|
||||
WHERE s.id IN (:streamIds)
|
||||
AND s.cleanup_pending = true
|
||||
AND NOT EXISTS (SELECT 1 FROM packet p WHERE p.stream_id = s.id)
|
||||
""", nativeQuery = true)
|
||||
int deleteFinishedStreams(@Param("streamIds") List<Long> streamIds);
|
||||
|
||||
@Query("SELECT p FROM Packet p " +
|
||||
"LEFT JOIN FETCH p.matches " +
|
||||
"WHERE p.stream.id = :streamId " +
|
||||
"AND p.stream.cleanupPending = false " +
|
||||
"AND (:startingFrom IS NULL OR p.id > :startingFrom) " +
|
||||
"ORDER BY p.id"
|
||||
)
|
||||
List<Packet> getPackets(long streamId, Long startingFrom, Pageable pageable);
|
||||
|
||||
interface CleanupCandidate {
|
||||
Long getId();
|
||||
|
||||
Long getPacketsCount();
|
||||
|
||||
Long getSizeBytes();
|
||||
|
||||
Integer getAttempts();
|
||||
}
|
||||
|
||||
interface CleanupPacket {
|
||||
Long getId();
|
||||
|
||||
Long getSizeBytes();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -28,7 +28,7 @@ public class CountingService {
|
||||
this.subscriptionService = subscriptionService;
|
||||
}
|
||||
|
||||
void countStream(int serviceId, int packets) {
|
||||
synchronized void countStream(int serviceId, int packets) {
|
||||
getCounter(servicesPackets, serviceId).increment(packets);
|
||||
getCounter(servicesStreams, serviceId).increment();
|
||||
|
||||
@ -38,15 +38,17 @@ public class CountingService {
|
||||
|
||||
@Scheduled(cron = "0 * * ? * *")
|
||||
public void sendCounters() {
|
||||
subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.COUNTERS_UPDATE,
|
||||
new CountersHolder(
|
||||
CountersHolder snapshot;
|
||||
synchronized (this) {
|
||||
snapshot = new CountersHolder(
|
||||
toIntegerMap(servicesPackets), toIntegerMap(servicesStreams),
|
||||
totalPackets.getValue(), totalStreams.getValue())));
|
||||
|
||||
servicesPackets.clear();
|
||||
servicesStreams.clear();
|
||||
totalPackets = new Counter();
|
||||
totalStreams = new Counter();
|
||||
totalPackets.getValue(), totalStreams.getValue());
|
||||
servicesPackets.clear();
|
||||
servicesStreams.clear();
|
||||
totalPackets = new Counter();
|
||||
totalStreams = new Counter();
|
||||
}
|
||||
subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.COUNTERS_UPDATE, snapshot));
|
||||
}
|
||||
|
||||
private Map<Integer, Integer> toIntegerMap(Map<Integer, Counter> map) {
|
||||
|
||||
@ -0,0 +1,245 @@
|
||||
package ru.serega6531.packmate.service;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Propagation;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import ru.serega6531.packmate.repository.StreamRepository;
|
||||
import ru.serega6531.packmate.repository.StreamRepository.CleanupCandidate;
|
||||
import ru.serega6531.packmate.repository.StreamRepository.CleanupPacket;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class OldStreamsCleaner {
|
||||
|
||||
private final StreamRepository repository;
|
||||
|
||||
public OldStreamsCleaner(StreamRepository repository) {
|
||||
this.repository = repository;
|
||||
}
|
||||
|
||||
@Transactional(propagation = Propagation.REQUIRES_NEW)
|
||||
public CleanupBatchResult cleanupBatch(long threshold, long persistedThreshold, CleanupBatchPolicy policy) {
|
||||
long batchDeadline = deadlineAfter(policy.statementTimeout());
|
||||
configureRemainingTimeout(batchDeadline, policy.lockTimeout());
|
||||
|
||||
long retryNow = System.currentTimeMillis();
|
||||
List<CleanupCandidate> candidates = repository.findAndLockPendingStreams(retryNow, policy.maxStreams());
|
||||
boolean claimed = false;
|
||||
if (candidates.isEmpty()) {
|
||||
configureRemainingTimeout(batchDeadline, policy.lockTimeout());
|
||||
candidates = repository.findAndLockOldStreamCandidates(
|
||||
threshold,
|
||||
persistedThreshold,
|
||||
retryNow,
|
||||
policy.maxStreams()
|
||||
);
|
||||
claimed = true;
|
||||
}
|
||||
|
||||
List<Long> selectedIds = selectWithinBudget(candidates, policy);
|
||||
if (selectedIds.isEmpty()) {
|
||||
return CleanupBatchResult.empty();
|
||||
}
|
||||
|
||||
try {
|
||||
if (claimed) {
|
||||
configureRemainingTimeout(batchDeadline, policy.lockTimeout());
|
||||
int marked = repository.markCleanupPending(selectedIds);
|
||||
if (marked != selectedIds.size()) {
|
||||
throw new IllegalStateException(
|
||||
"Expected to claim " + selectedIds.size() + " streams, claimed " + marked
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
configureRemainingTimeout(batchDeadline, policy.lockTimeout());
|
||||
int patternLinks = repository.deletePatternLinks(selectedIds);
|
||||
configureRemainingTimeout(batchDeadline, policy.lockTimeout());
|
||||
List<Long> packetIds = selectPacketsWithinBudget(
|
||||
repository.findPacketsForCleanup(selectedIds, Math.toIntExact(policy.maxPackets())),
|
||||
policy.maxBytes()
|
||||
);
|
||||
|
||||
int foundPatterns = 0;
|
||||
int packets = 0;
|
||||
if (!packetIds.isEmpty()) {
|
||||
configureRemainingTimeout(batchDeadline, policy.lockTimeout());
|
||||
List<Long> matchIds = repository.findFoundPatternsForCleanup(packetIds, policy.maxMatches());
|
||||
if (!matchIds.isEmpty()) {
|
||||
configureRemainingTimeout(batchDeadline, policy.lockTimeout());
|
||||
foundPatterns = repository.deleteFoundPatterns(matchIds);
|
||||
}
|
||||
configureRemainingTimeout(batchDeadline, policy.lockTimeout());
|
||||
packets = repository.deletePacketsWithoutMatches(packetIds);
|
||||
}
|
||||
|
||||
// A concurrent lookback that started before the claim may have committed a link.
|
||||
configureRemainingTimeout(batchDeadline, policy.lockTimeout());
|
||||
patternLinks += repository.deletePatternLinks(selectedIds);
|
||||
configureRemainingTimeout(batchDeadline, policy.lockTimeout());
|
||||
int streams = repository.deleteFinishedStreams(selectedIds);
|
||||
configureRemainingTimeout(batchDeadline, policy.lockTimeout());
|
||||
repository.resetCleanupRetry(selectedIds);
|
||||
|
||||
return new CleanupBatchResult(streams, packets, foundPatterns, patternLinks);
|
||||
} catch (RuntimeException e) {
|
||||
throw new CleanupBatchException(selectedIds, e);
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional(propagation = Propagation.REQUIRES_NEW)
|
||||
public void deferCleanup(List<Long> streamIds, Instant retryAfter) {
|
||||
int deferred = repository.deferCleanup(streamIds, retryAfter.toEpochMilli());
|
||||
log.warn("Deferred {} cleanup streams until {} after a failed batch", deferred, retryAfter);
|
||||
}
|
||||
|
||||
private long deadlineAfter(Duration timeout) {
|
||||
long now = System.nanoTime();
|
||||
long timeoutNanos = timeout.toNanos();
|
||||
return timeoutNanos >= Long.MAX_VALUE - now ? Long.MAX_VALUE : now + timeoutNanos;
|
||||
}
|
||||
|
||||
private void configureRemainingTimeout(long deadline, Duration lockTimeout) {
|
||||
long remainingNanos = deadline - System.nanoTime();
|
||||
if (remainingNanos <= 0) {
|
||||
throw new IllegalStateException("Cleanup batch exceeded its transaction deadline");
|
||||
}
|
||||
|
||||
long remainingMillis = Math.max(1, TimeUnit.NANOSECONDS.toMillis(remainingNanos));
|
||||
repository.configureLocalTimeouts(
|
||||
remainingMillis,
|
||||
Math.min(lockTimeout.toMillis(), remainingMillis)
|
||||
);
|
||||
}
|
||||
|
||||
private List<Long> selectWithinBudget(List<CleanupCandidate> candidates, CleanupBatchPolicy policy) {
|
||||
List<Long> result = new ArrayList<>(candidates.size());
|
||||
long packets = 0;
|
||||
long bytes = 0;
|
||||
|
||||
for (CleanupCandidate candidate : candidates) {
|
||||
long candidatePackets = nonNegative(candidate.getPacketsCount());
|
||||
long candidateBytes = nonNegative(candidate.getSizeBytes());
|
||||
long nextPackets = packets + candidatePackets;
|
||||
long nextBytes = bytes + candidateBytes;
|
||||
|
||||
if (!result.isEmpty()
|
||||
&& (nextPackets > policy.maxPackets() || nextBytes > policy.maxBytes())) {
|
||||
break;
|
||||
}
|
||||
|
||||
result.add(candidate.getId());
|
||||
packets = nextPackets;
|
||||
bytes = nextBytes;
|
||||
|
||||
if (packets > policy.maxPackets() || bytes > policy.maxBytes()) {
|
||||
log.warn(
|
||||
"Old stream {} exceeds one cleanup batch budget: packets={}, bytes={}",
|
||||
candidate.getId(), candidatePackets, candidateBytes
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private List<Long> selectPacketsWithinBudget(List<CleanupPacket> packets, long maxBytes) {
|
||||
List<Long> result = new ArrayList<>(packets.size());
|
||||
long bytes = 0;
|
||||
|
||||
for (CleanupPacket packet : packets) {
|
||||
long packetBytes = nonNegative(packet.getSizeBytes());
|
||||
if (!result.isEmpty() && bytes + packetBytes > maxBytes) {
|
||||
break;
|
||||
}
|
||||
|
||||
result.add(packet.getId());
|
||||
bytes += packetBytes;
|
||||
|
||||
if (bytes > maxBytes) {
|
||||
log.warn("Packet {} exceeds one cleanup byte budget: bytes={}", packet.getId(), packetBytes);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private long nonNegative(Long value) {
|
||||
return value == null ? 0 : Math.max(0, value);
|
||||
}
|
||||
|
||||
public record CleanupBatchPolicy(
|
||||
int maxStreams,
|
||||
long maxPackets,
|
||||
int maxMatches,
|
||||
long maxBytes,
|
||||
Duration statementTimeout,
|
||||
Duration lockTimeout
|
||||
) {
|
||||
public CleanupBatchPolicy {
|
||||
requirePositive(maxStreams, "maxStreams");
|
||||
requirePositive(maxPackets, "maxPackets");
|
||||
if (maxPackets > Integer.MAX_VALUE) {
|
||||
throw new IllegalArgumentException("maxPackets must not exceed " + Integer.MAX_VALUE);
|
||||
}
|
||||
requirePositive(maxMatches, "maxMatches");
|
||||
requirePositive(maxBytes, "maxBytes");
|
||||
requirePositive(statementTimeout.toMillis(), "statementTimeout");
|
||||
requirePositive(lockTimeout.toMillis(), "lockTimeout");
|
||||
}
|
||||
|
||||
public CleanupBatchPolicy withMaxStreams(int newMaxStreams) {
|
||||
return new CleanupBatchPolicy(
|
||||
newMaxStreams,
|
||||
maxPackets,
|
||||
maxMatches,
|
||||
maxBytes,
|
||||
statementTimeout,
|
||||
lockTimeout
|
||||
);
|
||||
}
|
||||
|
||||
private static void requirePositive(long value, String name) {
|
||||
if (value <= 0) {
|
||||
throw new IllegalArgumentException(name + " must be greater than zero");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public record CleanupBatchResult(
|
||||
int streams,
|
||||
int packets,
|
||||
int foundPatterns,
|
||||
int patternLinks
|
||||
) {
|
||||
public static CleanupBatchResult empty() {
|
||||
return new CleanupBatchResult(0, 0, 0, 0);
|
||||
}
|
||||
|
||||
public boolean hasWork() {
|
||||
return streams > 0 || packets > 0 || foundPatterns > 0 || patternLinks > 0;
|
||||
}
|
||||
}
|
||||
|
||||
public static final class CleanupBatchException extends RuntimeException {
|
||||
private final List<Long> streamIds;
|
||||
|
||||
public CleanupBatchException(List<Long> streamIds, RuntimeException cause) {
|
||||
super("Cleanup batch failed for streams " + streamIds, cause);
|
||||
this.streamIds = List.copyOf(streamIds);
|
||||
}
|
||||
|
||||
public List<Long> streamIds() {
|
||||
return streamIds;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -14,13 +14,16 @@ import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.io.StringReader;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.regex.Matcher;
|
||||
|
||||
public class PatternMatcher {
|
||||
|
||||
private static final Map<String, java.util.regex.Pattern> compiledRegexes = new HashMap<>();
|
||||
private static final Map<String, KMPStringSearcher> compiledStringKmps = new HashMap<>();
|
||||
private static final Map<String, KMPByteSearcher> compiledByteKmps = new HashMap<>();
|
||||
private static final Map<String, java.util.regex.Pattern> compiledRegexes = new ConcurrentHashMap<>();
|
||||
private static final ThreadLocal<Map<String, KMPStringSearcher>> compiledStringKmps =
|
||||
ThreadLocal.withInitial(HashMap::new);
|
||||
private static final ThreadLocal<Map<String, KMPByteSearcher>> compiledByteKmps =
|
||||
ThreadLocal.withInitial(HashMap::new);
|
||||
|
||||
private final byte[] contentBytes;
|
||||
private final String content;
|
||||
@ -177,11 +180,13 @@ public class PatternMatcher {
|
||||
}
|
||||
|
||||
private static KMPStringSearcher compileStringKMP(Pattern pattern) {
|
||||
return compiledStringKmps.computeIfAbsent(pattern.getValue(), val -> new KMPStringSearcher(val.toCharArray()));
|
||||
return compiledStringKmps.get().computeIfAbsent(
|
||||
pattern.getValue(), val -> new KMPStringSearcher(val.toCharArray()));
|
||||
}
|
||||
|
||||
private static KMPByteSearcher compileByteKMP(Pattern pattern) {
|
||||
return compiledByteKmps.computeIfAbsent(pattern.getValue(), val -> new KMPByteSearcher(Hex.decode(val)));
|
||||
return compiledByteKmps.get().computeIfAbsent(
|
||||
pattern.getValue(), val -> new KMPByteSearcher(Hex.decode(val)));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -22,10 +22,10 @@ import ru.serega6531.packmate.repository.PatternRepository;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
@Service
|
||||
@ -37,7 +37,7 @@ public class PatternService {
|
||||
private final SubscriptionService subscriptionService;
|
||||
private final ModelMapper modelMapper;
|
||||
|
||||
private final Map<Integer, Pattern> patterns = new HashMap<>();
|
||||
private final Map<Integer, Pattern> patterns = new ConcurrentHashMap<>();
|
||||
|
||||
@Autowired
|
||||
public PatternService(PatternRepository repository,
|
||||
@ -151,12 +151,13 @@ public class PatternService {
|
||||
long end = pattern.getSearchStartTimestamp();
|
||||
long start = end - TimeUnit.MINUTES.toMillis(minutes);
|
||||
|
||||
pattern.setSearchStartTimestamp(start);
|
||||
repository.save(pattern);
|
||||
|
||||
log.info("Scanning for pattern '{}' between {} and {}", pattern.getName(),
|
||||
Instant.ofEpochMilli(start), Instant.ofEpochMilli(end));
|
||||
streamService.processLookbackPattern(pattern, start, end);
|
||||
|
||||
// Only move the cursor after the bounded executor accepted the job.
|
||||
pattern.setSearchStartTimestamp(start);
|
||||
repository.save(pattern);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -6,6 +6,7 @@ import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import ru.serega6531.packmate.model.enums.SubscriptionMessageType;
|
||||
import ru.serega6531.packmate.model.pojo.ServiceDto;
|
||||
import ru.serega6531.packmate.model.pojo.PcapMetrics;
|
||||
import ru.serega6531.packmate.model.pojo.SubscriptionMessage;
|
||||
import ru.serega6531.packmate.pcap.NoOpPcapWorker;
|
||||
import ru.serega6531.packmate.pcap.PcapWorker;
|
||||
@ -64,4 +65,8 @@ public class PcapService {
|
||||
return worker.getExecutorState();
|
||||
}
|
||||
|
||||
public PcapMetrics getMetrics() {
|
||||
return worker.getMetrics();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -17,10 +17,10 @@ import ru.serega6531.packmate.model.pojo.SubscriptionMessage;
|
||||
import ru.serega6531.packmate.repository.ServiceRepository;
|
||||
|
||||
import java.net.InetAddress;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
@ -32,7 +32,7 @@ public class ServicesService {
|
||||
|
||||
private final InetAddress localIp;
|
||||
|
||||
private final Map<Integer, CtfService> services = new HashMap<>();
|
||||
private final Map<Integer, CtfService> services = new ConcurrentHashMap<>();
|
||||
private final ModelMapper modelMapper;
|
||||
|
||||
@Autowired
|
||||
|
||||
@ -30,8 +30,8 @@ import ru.serega6531.packmate.repository.StreamRepository;
|
||||
import ru.serega6531.packmate.service.optimization.RsaKeysHolder;
|
||||
import ru.serega6531.packmate.service.optimization.StreamOptimizer;
|
||||
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
@ -101,9 +101,6 @@ public class StreamService {
|
||||
|
||||
countingService.countStream(service.getPort(), packets.size());
|
||||
|
||||
int packetsSize = packets.stream().mapToInt(p -> p.getContent().length).sum();
|
||||
int packetsCount = packets.size();
|
||||
|
||||
List<Packet> optimizedPackets = new StreamOptimizer(keysHolder, service, packets).optimizeStream();
|
||||
|
||||
if (isStreamIgnored(optimizedPackets, service)) {
|
||||
@ -125,9 +122,11 @@ public class StreamService {
|
||||
String userAgentHash = getUserAgentHash(optimizedPackets);
|
||||
stream.setUserAgentHash(userAgentHash);
|
||||
|
||||
stream.setSizeBytes(packetsSize);
|
||||
stream.setPacketsCount(packetsCount);
|
||||
|
||||
long optimizedSize = optimizedPackets.stream()
|
||||
.mapToLong(packet -> packet.getContent().length)
|
||||
.sum();
|
||||
stream.setSizeBytes((int) Math.min(Integer.MAX_VALUE, optimizedSize));
|
||||
stream.setPacketsCount(optimizedPackets.size());
|
||||
Set<Pattern> foundPatterns = matchPatterns(optimizedPackets, service);
|
||||
stream.setFoundPatterns(foundPatterns);
|
||||
stream.setPackets(optimizedPackets);
|
||||
@ -136,13 +135,14 @@ public class StreamService {
|
||||
packet.setStream(stream);
|
||||
}
|
||||
|
||||
stream.setPersistedAt(System.currentTimeMillis());
|
||||
Stream savedStream = save(stream);
|
||||
|
||||
subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.NEW_STREAM, streamToDto(savedStream)));
|
||||
return true;
|
||||
}
|
||||
|
||||
@Async
|
||||
@Async("lookbackExecutor")
|
||||
@Transactional
|
||||
public void processLookbackPattern(Pattern pattern, long start, long end) {
|
||||
List<Stream> streams = findAllBetweenTimestamps(start, end);
|
||||
@ -264,24 +264,24 @@ public class StreamService {
|
||||
.toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Number of deleted rows
|
||||
*/
|
||||
@Transactional
|
||||
public long cleanupOldStreams(ZonedDateTime before) {
|
||||
return repository.deleteByEndTimestampBeforeAndFavoriteIsFalse(before.toEpochSecond() * 1000);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void setFavorite(long id, boolean favorite) {
|
||||
repository.setFavorite(id, favorite);
|
||||
}
|
||||
|
||||
@Transactional(readOnly = true)
|
||||
public List<Long> findAvailableStreamIds(List<Long> ids) {
|
||||
if (ids.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
return repository.findAvailableStreamIds(new java.util.ArrayList<>(new LinkedHashSet<>(ids)));
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public List<StreamDto> findAll(StreamPagination pagination, Optional<Integer> service, boolean onlyFavorites) {
|
||||
PageRequest page = PageRequest.of(0, pagination.getPageSize(), Sort.Direction.DESC, "id");
|
||||
|
||||
Specification<Stream> spec = Specification.where(null);
|
||||
Specification<Stream> spec = Specification.where(streamCleanupPendingIsFalse());
|
||||
|
||||
if (pagination.getStartingFrom() != null) {
|
||||
spec = spec.and(streamIdLessThan(pagination.getStartingFrom()));
|
||||
@ -307,7 +307,8 @@ public class StreamService {
|
||||
}
|
||||
|
||||
public List<Stream> findAllBetweenTimestamps(long start, long end) {
|
||||
Specification<Stream> spec = streamTimestampBetween(start, end);
|
||||
Specification<Stream> spec = streamTimestampBetween(start, end)
|
||||
.and(streamCleanupPendingIsFalse());
|
||||
return repository.findAll(spec);
|
||||
}
|
||||
|
||||
@ -327,6 +328,10 @@ public class StreamService {
|
||||
return (root, query, cb) -> cb.equal(root.get("favorite"), true);
|
||||
}
|
||||
|
||||
private Specification<Stream> streamCleanupPendingIsFalse() {
|
||||
return (root, query, cb) -> cb.equal(root.get("cleanupPending"), false);
|
||||
}
|
||||
|
||||
private Specification<Stream> streamIdLessThan(long id) {
|
||||
return (root, query, cb) -> cb.lessThan(root.get("id"), id);
|
||||
}
|
||||
|
||||
@ -53,8 +53,8 @@ public class RsaKeysHolder {
|
||||
keys.put(privateKey.getModulus(), privateKey);
|
||||
String n = privateKey.getModulus().toString();
|
||||
log.info("Loaded RSA key with N={}...", n.substring(0, Math.min(n.length(), 8)));
|
||||
} catch (IOException | InvalidKeySpecException e) {
|
||||
log.error("Error loading rsa key", e);
|
||||
} catch (IOException | InvalidKeySpecException | IllegalArgumentException e) {
|
||||
log.error("Error loading RSA key from {}", keyFile, e);
|
||||
}
|
||||
}
|
||||
|
||||
@ -63,7 +63,7 @@ public class RsaKeysHolder {
|
||||
|
||||
content = content.replaceAll("-----BEGIN (RSA )?PRIVATE KEY-----", "")
|
||||
.replaceAll("-----END (RSA )?PRIVATE KEY-----", "")
|
||||
.replace("\n", "");
|
||||
.replaceAll("\\s", "");
|
||||
|
||||
byte[] keyBytes = Base64.getDecoder().decode(content);
|
||||
|
||||
|
||||
@ -5,29 +5,187 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
import ru.serega6531.packmate.properties.PackmateProperties;
|
||||
import ru.serega6531.packmate.service.StreamService;
|
||||
import ru.serega6531.packmate.model.enums.SubscriptionMessageType;
|
||||
import ru.serega6531.packmate.model.pojo.SubscriptionMessage;
|
||||
import ru.serega6531.packmate.service.OldStreamsCleaner;
|
||||
import ru.serega6531.packmate.service.SubscriptionService;
|
||||
import ru.serega6531.packmate.service.OldStreamsCleaner.CleanupBatchException;
|
||||
import ru.serega6531.packmate.service.OldStreamsCleaner.CleanupBatchResult;
|
||||
import ru.serega6531.packmate.service.OldStreamsCleaner.CleanupBatchPolicy;
|
||||
|
||||
import java.time.ZonedDateTime;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
@ConditionalOnExpression("${packmate.cleanup.enabled:false} && '${packmate.capture-mode}' == 'LIVE'")
|
||||
public class OldStreamsCleanupTask {
|
||||
|
||||
private final StreamService service;
|
||||
private final OldStreamsCleaner cleaner;
|
||||
private final SubscriptionService subscriptionService;
|
||||
private final int oldStreamsThreshold;
|
||||
private final int intervalSeconds;
|
||||
private final int maxBatchesPerRun;
|
||||
private final Duration maxRunDuration;
|
||||
private final Duration commitGrace;
|
||||
private final CleanupBatchPolicy batchPolicy;
|
||||
|
||||
public OldStreamsCleanupTask(StreamService service, PackmateProperties properties) {
|
||||
this.service = service;
|
||||
this.oldStreamsThreshold = properties.cleanup().threshold();
|
||||
public OldStreamsCleanupTask(OldStreamsCleaner cleaner,
|
||||
PackmateProperties properties,
|
||||
SubscriptionService subscriptionService) {
|
||||
this.cleaner = cleaner;
|
||||
this.subscriptionService = subscriptionService;
|
||||
PackmateProperties.CleanupProperties cleanup = properties.cleanup();
|
||||
this.oldStreamsThreshold = requirePositive(cleanup.threshold(), "threshold");
|
||||
this.intervalSeconds = requirePositive(cleanup.intervalSeconds(), "interval-seconds");
|
||||
this.maxBatchesPerRun = requirePositive(cleanup.maxBatchesPerRun(), "max-batches-per-run");
|
||||
this.maxRunDuration = Duration.ofSeconds(requirePositive(cleanup.maxRunSeconds(), "max-run-seconds"));
|
||||
this.commitGrace = Duration.ofSeconds(requireNonNegative(
|
||||
cleanup.commitGraceSeconds(),
|
||||
"commit-grace-seconds"
|
||||
));
|
||||
this.batchPolicy = new CleanupBatchPolicy(
|
||||
cleanup.batchSize(),
|
||||
cleanup.maxPacketsPerBatch(),
|
||||
cleanup.maxMatchesPerBatch(),
|
||||
cleanup.maxBytesPerBatch(),
|
||||
Duration.ofSeconds(cleanup.statementTimeoutSeconds()),
|
||||
Duration.ofSeconds(cleanup.lockTimeoutSeconds())
|
||||
);
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelayString = "PT${packmate.cleanup.interval}M", initialDelayString = "PT1M")
|
||||
@Scheduled(fixedDelayString = "PT${packmate.cleanup.interval-seconds}S", initialDelayString = "PT10S")
|
||||
public void cleanup() {
|
||||
ZonedDateTime before = ZonedDateTime.now().minusMinutes(oldStreamsThreshold);
|
||||
log.info("Cleaning up old non-favorite streams (before {})", before);
|
||||
long deleted = service.cleanupOldStreams(before);
|
||||
log.info("Deleted {} rows", deleted);
|
||||
runCleanup();
|
||||
}
|
||||
|
||||
public CleanupRunResult runCleanup() {
|
||||
Instant now = Instant.now();
|
||||
long endTimestampThreshold = now.minus(Duration.ofMinutes(oldStreamsThreshold)).toEpochMilli();
|
||||
long persistedThreshold = now.minus(commitGrace).toEpochMilli();
|
||||
long startedAt = System.nanoTime();
|
||||
long deadline = startedAt + maxRunDuration.toNanos();
|
||||
long streams = 0;
|
||||
long packets = 0;
|
||||
long foundPatterns = 0;
|
||||
long patternLinks = 0;
|
||||
long maxBatchNanos = 0;
|
||||
int batches = 0;
|
||||
|
||||
boolean failed = false;
|
||||
boolean moreWork = false;
|
||||
boolean invalidationNeeded = false;
|
||||
CleanupBatchPolicy activePolicy = batchPolicy;
|
||||
while (batches < maxBatchesPerRun && System.nanoTime() < deadline) {
|
||||
CleanupBatchResult result;
|
||||
long batchStartedAt = System.nanoTime();
|
||||
try {
|
||||
result = cleaner.cleanupBatch(endTimestampThreshold, persistedThreshold, activePolicy);
|
||||
} catch (RuntimeException e) {
|
||||
maxBatchNanos = Math.max(maxBatchNanos, System.nanoTime() - batchStartedAt);
|
||||
if (e instanceof CleanupBatchException batchException) {
|
||||
if (activePolicy.maxStreams() > 1 && batchException.streamIds().size() > 1) {
|
||||
activePolicy = batchPolicy.withMaxStreams(1);
|
||||
log.warn(
|
||||
"Cleanup batch failed for {} streams; isolating retries to one stream per transaction",
|
||||
batchException.streamIds().size(),
|
||||
e
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
failed = true;
|
||||
Instant retryAfter = Instant.now().plusSeconds(Math.max(60L, intervalSeconds * 6L));
|
||||
try {
|
||||
cleaner.deferCleanup(batchException.streamIds(), retryAfter);
|
||||
} catch (RuntimeException deferFailure) {
|
||||
e.addSuppressed(deferFailure);
|
||||
log.error("Old streams cleanup batch and retry deferral failed", e);
|
||||
break;
|
||||
}
|
||||
log.error("Old streams cleanup batch failed; affected streams were deferred", e);
|
||||
moreWork = true;
|
||||
continue;
|
||||
}
|
||||
failed = true;
|
||||
log.error("Old streams cleanup batch failed; it will be retried on the next run", e);
|
||||
break;
|
||||
}
|
||||
maxBatchNanos = Math.max(maxBatchNanos, System.nanoTime() - batchStartedAt);
|
||||
|
||||
moreWork = result.hasWork();
|
||||
if (!moreWork) {
|
||||
break;
|
||||
}
|
||||
invalidationNeeded = true;
|
||||
|
||||
batches++;
|
||||
streams += result.streams();
|
||||
packets += result.packets();
|
||||
foundPatterns += result.foundPatterns();
|
||||
patternLinks += result.patternLinks();
|
||||
}
|
||||
|
||||
long elapsedMillis = Duration.ofNanos(System.nanoTime() - startedAt).toMillis();
|
||||
log.info(
|
||||
"Old streams cleanup finished: streams={}, packets={}, matches={}, patternLinks={}, batches={}, "
|
||||
+ "elapsedMs={}, maxBatchMs={}, failed={}",
|
||||
streams, packets, foundPatterns, patternLinks, batches, elapsedMillis,
|
||||
Duration.ofNanos(maxBatchNanos).toMillis(), failed
|
||||
);
|
||||
|
||||
if (!failed && moreWork && (batches == maxBatchesPerRun || System.nanoTime() >= deadline)) {
|
||||
log.info("Old streams cleanup yielded after reaching its run budget; remaining rows will resume in {} seconds",
|
||||
intervalSeconds);
|
||||
}
|
||||
|
||||
if (invalidationNeeded) {
|
||||
try {
|
||||
subscriptionService.broadcast(new SubscriptionMessage(
|
||||
SubscriptionMessageType.STREAMS_INVALIDATED,
|
||||
null
|
||||
));
|
||||
} catch (RuntimeException e) {
|
||||
log.warn("Cleanup committed, but the UI invalidation event could not be sent", e);
|
||||
}
|
||||
}
|
||||
|
||||
return new CleanupRunResult(
|
||||
streams,
|
||||
packets,
|
||||
foundPatterns,
|
||||
patternLinks,
|
||||
batches,
|
||||
elapsedMillis,
|
||||
maxBatchNanos,
|
||||
moreWork,
|
||||
failed
|
||||
);
|
||||
}
|
||||
|
||||
private static int requirePositive(int value, String property) {
|
||||
if (value <= 0) {
|
||||
throw new IllegalArgumentException("packmate.cleanup." + property + " must be greater than zero");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private static int requireNonNegative(int value, String property) {
|
||||
if (value < 0) {
|
||||
throw new IllegalArgumentException("packmate.cleanup." + property + " must not be negative");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
public record CleanupRunResult(
|
||||
long streams,
|
||||
long packets,
|
||||
long foundPatterns,
|
||||
long patternLinks,
|
||||
int batches,
|
||||
long elapsedMillis,
|
||||
long maxBatchNanos,
|
||||
boolean moreWork,
|
||||
boolean failed
|
||||
) {}
|
||||
}
|
||||
|
||||
@ -31,8 +31,22 @@ packmate:
|
||||
udp-stream-timeout: 20 # seconds
|
||||
tcp-stream-timeout: 40 # seconds
|
||||
check-interval: 10 # seconds
|
||||
capture:
|
||||
workers: 1 # compatibility mode; increase after running the network stress profile
|
||||
queue-capacity: 250000 # total bounded queue across all workers
|
||||
buffer-size-bytes: 67108864 # 64 MiB kernel capture buffer for bursts
|
||||
shutdown-timeout-seconds: 30
|
||||
cleanup:
|
||||
enabled: true
|
||||
threshold: 240 # minutes
|
||||
interval: 5 # minutes
|
||||
ignore-empty-packets: true
|
||||
threshold: 15 # minutes
|
||||
interval-seconds: 10
|
||||
max-run-seconds: 5
|
||||
commit-grace-seconds: 120
|
||||
batch-size: 500
|
||||
max-batches-per-run: 20
|
||||
max-packets-per-batch: 20000
|
||||
max-matches-per-batch: 20000
|
||||
max-bytes-per-batch: 67108864 # 64 MiB
|
||||
statement-timeout-seconds: 5
|
||||
lock-timeout-seconds: 1
|
||||
ignore-empty-packets: true
|
||||
|
||||
@ -0,0 +1,60 @@
|
||||
package ru.serega6531.packmate.pcap;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import ru.serega6531.packmate.model.enums.Protocol;
|
||||
import ru.serega6531.packmate.model.pojo.UnfinishedStream;
|
||||
|
||||
import java.net.InetAddress;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class AbstractPcapWorkerShardingTest {
|
||||
|
||||
@Test
|
||||
void bothDirectionsOfAFlowUseTheSameProcessor() throws Exception {
|
||||
InetAddress client = InetAddress.getByName("172.31.250.101");
|
||||
InetAddress server = InetAddress.getByName("172.31.250.10");
|
||||
|
||||
for (int processorCount : new int[]{1, 2, 4, 8, 16}) {
|
||||
UnfinishedStream request = new UnfinishedStream(client, server, 43123, 18080, Protocol.TCP);
|
||||
UnfinishedStream response = new UnfinishedStream(server, client, 18080, 43123, Protocol.TCP);
|
||||
assertEquals(
|
||||
AbstractPcapWorker.processorIndex(request, processorCount),
|
||||
AbstractPcapWorker.processorIndex(response, processorCount)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void flowIdentityKeepsIpAndPortPairedAsAnEndpoint() throws Exception {
|
||||
InetAddress client = InetAddress.getByName("172.31.250.101");
|
||||
InetAddress server = InetAddress.getByName("172.31.250.10");
|
||||
UnfinishedStream request = new UnfinishedStream(client, server, 43123, 18080, Protocol.TCP);
|
||||
UnfinishedStream response = new UnfinishedStream(server, client, 18080, 43123, Protocol.TCP);
|
||||
UnfinishedStream crossedPorts = new UnfinishedStream(client, server, 18080, 43123, Protocol.TCP);
|
||||
|
||||
assertEquals(request, response);
|
||||
assertEquals(request.hashCode(), response.hashCode());
|
||||
assertNotEquals(request, crossedPorts);
|
||||
}
|
||||
|
||||
@Test
|
||||
void independentFlowsAreDistributedAcrossProcessors() throws Exception {
|
||||
InetAddress server = InetAddress.getByName("172.31.250.10");
|
||||
Set<Integer> processors = new HashSet<>();
|
||||
|
||||
for (int host = 1; host <= 32; host++) {
|
||||
InetAddress client = InetAddress.getByName("172.31.251." + host);
|
||||
for (int port = 40000; port < 40100; port++) {
|
||||
UnfinishedStream stream = new UnfinishedStream(client, server, port, 18080, Protocol.TCP);
|
||||
processors.add(AbstractPcapWorker.processorIndex(stream, 8));
|
||||
}
|
||||
}
|
||||
|
||||
assertTrue(processors.size() >= 7, "flow hashing should use almost all configured processors");
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,110 @@
|
||||
package ru.serega6531.packmate.pcap;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import ru.serega6531.packmate.model.enums.Protocol;
|
||||
import ru.serega6531.packmate.model.pojo.PcapMetrics;
|
||||
import ru.serega6531.packmate.service.ServicesService;
|
||||
import ru.serega6531.packmate.service.StreamService;
|
||||
|
||||
import java.net.InetAddress;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
class AbstractPcapWorkerShutdownTest {
|
||||
|
||||
@Test
|
||||
void shutdownCancelsQueuedControlTaskAndUnblocksItsCaller() throws Exception {
|
||||
ThreadPoolExecutor processor = new ThreadPoolExecutor(
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
TimeUnit.MILLISECONDS,
|
||||
new ArrayBlockingQueue<>(1)
|
||||
);
|
||||
TestWorker worker = new TestWorker(processor);
|
||||
ExecutorService caller = Executors.newSingleThreadExecutor();
|
||||
CountDownLatch blockerStarted = new CountDownLatch(1);
|
||||
CountDownLatch releaseBlocker = new CountDownLatch(1);
|
||||
|
||||
try {
|
||||
processor.execute(() -> {
|
||||
blockerStarted.countDown();
|
||||
try {
|
||||
releaseBlocker.await();
|
||||
} catch (InterruptedException exception) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
});
|
||||
assertTrue(blockerStarted.await(1, TimeUnit.SECONDS));
|
||||
|
||||
Future<Integer> waitingControl = caller.submit(
|
||||
() -> worker.closeTimeoutStreams(Protocol.TCP, 1)
|
||||
);
|
||||
awaitQueueSize(processor, 1);
|
||||
|
||||
long shutdownStarted = System.nanoTime();
|
||||
assertFalse(worker.shutdown(1));
|
||||
long shutdownMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - shutdownStarted);
|
||||
assertTrue(shutdownMillis < 2_500,
|
||||
"shutdown enqueue must honor the shared deadline, took " + shutdownMillis + " ms");
|
||||
assertThrows(ExecutionException.class, () -> waitingControl.get(2, TimeUnit.SECONDS));
|
||||
assertThrows(RejectedExecutionException.class,
|
||||
() -> worker.closeTimeoutStreams(Protocol.TCP, 1));
|
||||
} finally {
|
||||
releaseBlocker.countDown();
|
||||
caller.shutdownNow();
|
||||
processor.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
private static void awaitQueueSize(ThreadPoolExecutor executor, int expected) throws InterruptedException {
|
||||
long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1);
|
||||
while (executor.getQueue().size() < expected && System.nanoTime() < deadline) {
|
||||
Thread.sleep(10);
|
||||
}
|
||||
assertTrue(executor.getQueue().size() >= expected, "control task was not queued in time");
|
||||
}
|
||||
|
||||
private static final class TestWorker extends AbstractPcapWorker {
|
||||
|
||||
private TestWorker(ThreadPoolExecutor executor) throws Exception {
|
||||
super(mock(ServicesService.class), mock(StreamService.class), InetAddress.getLoopbackAddress());
|
||||
initializeProcessorShards(List.of(executor));
|
||||
}
|
||||
|
||||
private boolean shutdown(int timeoutSeconds) {
|
||||
return flushAndShutdownProcessors(timeoutSeconds);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getExecutorState() {
|
||||
return "test";
|
||||
}
|
||||
|
||||
@Override
|
||||
public PcapMetrics getMetrics() {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,386 @@
|
||||
package ru.serega6531.packmate.service;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase;
|
||||
import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.dao.DataAccessException;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.test.context.DynamicPropertyRegistry;
|
||||
import org.springframework.test.context.DynamicPropertySource;
|
||||
import org.springframework.transaction.annotation.Propagation;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.testcontainers.containers.PostgreSQLContainer;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
import org.testcontainers.junit.jupiter.Testcontainers;
|
||||
import org.testcontainers.utility.DockerImageName;
|
||||
import ru.serega6531.packmate.configuration.CleanupDatabaseInitializer;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
import java.sql.Connection;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@DataJpaTest(showSql = false)
|
||||
@Import({OldStreamsCleaner.class, CleanupDatabaseInitializer.class})
|
||||
@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE)
|
||||
@Transactional(propagation = Propagation.NOT_SUPPORTED)
|
||||
@Testcontainers
|
||||
class OldStreamsCleanerPostgresTest {
|
||||
|
||||
@Container
|
||||
private static final PostgreSQLContainer<?> POSTGRES = new PostgreSQLContainer<>(
|
||||
DockerImageName.parse("postgres:15.2")
|
||||
)
|
||||
.withDatabaseName("packmate")
|
||||
.withUsername("packmate")
|
||||
.withPassword("test");
|
||||
|
||||
@DynamicPropertySource
|
||||
static void configureDatasource(DynamicPropertyRegistry registry) {
|
||||
registry.add("spring.datasource.url", POSTGRES::getJdbcUrl);
|
||||
registry.add("spring.datasource.username", POSTGRES::getUsername);
|
||||
registry.add("spring.datasource.password", POSTGRES::getPassword);
|
||||
registry.add("spring.jpa.hibernate.ddl-auto", () -> "create");
|
||||
}
|
||||
|
||||
@Autowired
|
||||
private JdbcTemplate jdbc;
|
||||
|
||||
@Autowired
|
||||
private OldStreamsCleaner cleaner;
|
||||
|
||||
@Autowired
|
||||
private CleanupDatabaseInitializer databaseInitializer;
|
||||
|
||||
@Autowired
|
||||
private DataSource dataSource;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
jdbc.execute("TRUNCATE TABLE stream_found_patterns, found_pattern, packet, stream, pattern CASCADE");
|
||||
|
||||
jdbc.update("""
|
||||
INSERT INTO pattern
|
||||
(id, enabled, deleted, name, value, color, search_type, direction_type,
|
||||
action_type, search_start_timestamp, search_target)
|
||||
VALUES (1, true, false, 'test', 'test', '#fff', 0, 0, 0, 0, 'CONTENT')
|
||||
""");
|
||||
|
||||
insertStream(1, 100, false);
|
||||
insertStream(2, 100, true);
|
||||
insertStream(3, 2_000, false);
|
||||
|
||||
insertPacket(11, 1);
|
||||
insertPacket(12, 2);
|
||||
insertPacket(13, 3);
|
||||
|
||||
jdbc.update("""
|
||||
INSERT INTO found_pattern
|
||||
(id, packet_id, pattern_id, search_target, start_position, end_position)
|
||||
VALUES (21, 11, 1, 'CONTENT', 0, 1)
|
||||
""");
|
||||
jdbc.update("INSERT INTO stream_found_patterns (stream_id, pattern_id) VALUES (1, 1)");
|
||||
}
|
||||
|
||||
@Test
|
||||
void deletesOnlyOldNonFavoriteStreamsWithTheirChildren() {
|
||||
OldStreamsCleaner.CleanupBatchPolicy policy = new OldStreamsCleaner.CleanupBatchPolicy(
|
||||
10, 1_000, 1_000, 1_000_000, Duration.ofSeconds(30), Duration.ofSeconds(2)
|
||||
);
|
||||
OldStreamsCleaner.CleanupBatchResult result = cleaner.cleanupBatch(1_000, Long.MAX_VALUE, policy);
|
||||
|
||||
assertEquals(new OldStreamsCleaner.CleanupBatchResult(1, 1, 1, 1), result);
|
||||
assertEquals(2, count("stream"));
|
||||
assertEquals(2, count("packet"));
|
||||
assertEquals(0, count("found_pattern"));
|
||||
assertEquals(0, count("stream_found_patterns"));
|
||||
assertEquals(1, countWhere("stream", "id = 2 AND favorite = true"));
|
||||
assertEquals(1, countWhere("stream", "id = 3"));
|
||||
assertEquals(1, countWhere("pg_indexes", "indexname = 'stream_cleanup_index'"));
|
||||
assertEquals(1, countWhere("pg_indexes", "indexname = 'stream_cleanup_pending_index'"));
|
||||
assertEquals(1, countWhere("pg_indexes", "indexname = 'packet_cleanup_stream_id_index'"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void resumesAStreamAcrossBoundedPacketChunksEvenWhenMetadataIsWrong() {
|
||||
jdbc.update("UPDATE stream SET favorite = true WHERE id = 1");
|
||||
insertStream(4, 50, false);
|
||||
insertPacket(41, 4);
|
||||
insertPacket(42, 4);
|
||||
insertPacket(43, 4);
|
||||
|
||||
OldStreamsCleaner.CleanupBatchPolicy policy = new OldStreamsCleaner.CleanupBatchPolicy(
|
||||
1, 2, 2, 2, Duration.ofSeconds(30), Duration.ofSeconds(2)
|
||||
);
|
||||
|
||||
OldStreamsCleaner.CleanupBatchResult first = cleaner.cleanupBatch(1_000, Long.MAX_VALUE, policy);
|
||||
assertEquals(0, first.streams());
|
||||
assertEquals(2, first.packets());
|
||||
assertEquals(1, countWhere("stream", "id = 4 AND cleanup_pending = true"));
|
||||
assertEquals(1, countWhere("packet", "stream_id = 4"));
|
||||
|
||||
OldStreamsCleaner.CleanupBatchResult second = cleaner.cleanupBatch(1_000, Long.MAX_VALUE, policy);
|
||||
assertEquals(1, second.streams());
|
||||
assertEquals(1, second.packets());
|
||||
assertEquals(0, countWhere("stream", "id = 4"));
|
||||
assertEquals(0, countWhere("packet", "stream_id = 4"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void skipsALockedHeadStreamAndCleansTheNextCandidate() throws Exception {
|
||||
insertStream(4, 200, false);
|
||||
insertPacket(41, 4);
|
||||
OldStreamsCleaner.CleanupBatchPolicy policy = new OldStreamsCleaner.CleanupBatchPolicy(
|
||||
1, 10, 10, 1_000, Duration.ofSeconds(30), Duration.ofSeconds(2)
|
||||
);
|
||||
|
||||
try (Connection blocker = dataSource.getConnection()) {
|
||||
blocker.setAutoCommit(false);
|
||||
blocker.createStatement().execute("SELECT id FROM stream WHERE id = 1 FOR UPDATE");
|
||||
|
||||
OldStreamsCleaner.CleanupBatchResult result = cleaner.cleanupBatch(
|
||||
1_000,
|
||||
Long.MAX_VALUE,
|
||||
policy
|
||||
);
|
||||
|
||||
assertEquals(1, result.streams());
|
||||
assertEquals(1, countWhere("stream", "id = 1"));
|
||||
assertEquals(0, countWhere("stream", "id = 4"));
|
||||
blocker.rollback();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void doesNotDeleteARecentlyCommittedStreamWithAnOldCaptureTimestamp() {
|
||||
jdbc.update("UPDATE stream SET favorite = true WHERE id = 1");
|
||||
insertStream(4, 50, false);
|
||||
insertPacket(41, 4);
|
||||
jdbc.update("UPDATE stream SET persisted_at = 5000 WHERE id = 4");
|
||||
OldStreamsCleaner.CleanupBatchPolicy policy = new OldStreamsCleaner.CleanupBatchPolicy(
|
||||
10, 10, 10, 1_000, Duration.ofSeconds(30), Duration.ofSeconds(2)
|
||||
);
|
||||
|
||||
OldStreamsCleaner.CleanupBatchResult result = cleaner.cleanupBatch(1_000, 4_000, policy);
|
||||
|
||||
assertEquals(OldStreamsCleaner.CleanupBatchResult.empty(), result);
|
||||
assertEquals(1, countWhere("stream", "id = 4 AND cleanup_pending = false"));
|
||||
assertEquals(1, countWhere("packet", "stream_id = 4"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void deletesDenseMatchesInBoundedChunksBeforeDeletingTheirPacket() {
|
||||
jdbc.update("UPDATE stream SET favorite = true WHERE id = 1");
|
||||
insertStream(4, 50, false);
|
||||
insertPacket(41, 4);
|
||||
jdbc.update("""
|
||||
INSERT INTO found_pattern
|
||||
(id, packet_id, pattern_id, search_target, start_position, end_position)
|
||||
VALUES
|
||||
(31, 41, 1, 'CONTENT', 0, 1),
|
||||
(32, 41, 1, 'CONTENT', 1, 2),
|
||||
(33, 41, 1, 'CONTENT', 2, 3)
|
||||
""");
|
||||
OldStreamsCleaner.CleanupBatchPolicy policy = new OldStreamsCleaner.CleanupBatchPolicy(
|
||||
1, 10, 2, 1_000, Duration.ofSeconds(5), Duration.ofSeconds(1)
|
||||
);
|
||||
|
||||
OldStreamsCleaner.CleanupBatchResult first = cleaner.cleanupBatch(1_000, Long.MAX_VALUE, policy);
|
||||
assertEquals(2, first.foundPatterns());
|
||||
assertEquals(0, first.packets());
|
||||
assertEquals(0, first.streams());
|
||||
assertEquals(1, countWhere("found_pattern", "packet_id = 41"));
|
||||
assertEquals(1, countWhere("packet", "id = 41"));
|
||||
|
||||
OldStreamsCleaner.CleanupBatchResult second = cleaner.cleanupBatch(1_000, Long.MAX_VALUE, policy);
|
||||
assertEquals(1, second.foundPatterns());
|
||||
assertEquals(1, second.packets());
|
||||
assertEquals(1, second.streams());
|
||||
assertEquals(0, countWhere("stream", "id = 4"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void enforcesOneDeadlineAcrossAllStatementsInABatch() {
|
||||
jdbc.execute("""
|
||||
CREATE OR REPLACE FUNCTION delay_cleanup_delete() RETURNS trigger
|
||||
LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF TG_TABLE_NAME = 'stream_found_patterns' THEN
|
||||
PERFORM pg_sleep(0.3);
|
||||
ELSE
|
||||
PERFORM pg_sleep(2);
|
||||
END IF;
|
||||
RETURN NULL;
|
||||
END
|
||||
$$
|
||||
""");
|
||||
jdbc.execute("""
|
||||
CREATE TRIGGER delay_pattern_link_cleanup
|
||||
BEFORE DELETE ON stream_found_patterns
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION delay_cleanup_delete()
|
||||
""");
|
||||
jdbc.execute("""
|
||||
CREATE TRIGGER delay_packet_cleanup
|
||||
BEFORE DELETE ON packet
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION delay_cleanup_delete()
|
||||
""");
|
||||
OldStreamsCleaner.CleanupBatchPolicy policy = new OldStreamsCleaner.CleanupBatchPolicy(
|
||||
1, 10, 10, 1_000, Duration.ofMillis(500), Duration.ofMillis(100)
|
||||
);
|
||||
|
||||
long startedAt = System.nanoTime();
|
||||
try {
|
||||
assertThrows(
|
||||
OldStreamsCleaner.CleanupBatchException.class,
|
||||
() -> cleaner.cleanupBatch(1_000, Long.MAX_VALUE, policy)
|
||||
);
|
||||
} finally {
|
||||
jdbc.execute("DROP TRIGGER delay_pattern_link_cleanup ON stream_found_patterns");
|
||||
jdbc.execute("DROP TRIGGER delay_packet_cleanup ON packet");
|
||||
jdbc.execute("DROP FUNCTION delay_cleanup_delete()");
|
||||
}
|
||||
long elapsedMillis = Duration.ofNanos(System.nanoTime() - startedAt).toMillis();
|
||||
|
||||
assertTrue(elapsedMillis < 1_500, "cleanup deadline took " + elapsedMillis + " ms");
|
||||
assertEquals(1, countWhere("stream", "id = 1 AND cleanup_pending = false"));
|
||||
assertEquals(1, countWhere("packet", "id = 11"));
|
||||
assertEquals(1, countWhere("stream_found_patterns", "stream_id = 1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rollsBackDefersAndResumesAfterADeleteFailure() {
|
||||
jdbc.execute("""
|
||||
CREATE OR REPLACE FUNCTION reject_packet_cleanup() RETURNS trigger
|
||||
LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
RAISE EXCEPTION 'forced packet cleanup failure';
|
||||
END
|
||||
$$
|
||||
""");
|
||||
jdbc.execute("""
|
||||
CREATE TRIGGER reject_packet_cleanup
|
||||
BEFORE DELETE ON packet
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION reject_packet_cleanup()
|
||||
""");
|
||||
|
||||
OldStreamsCleaner.CleanupBatchPolicy policy = new OldStreamsCleaner.CleanupBatchPolicy(
|
||||
10, 1_000, 1_000, 1_000_000, Duration.ofSeconds(5), Duration.ofSeconds(1)
|
||||
);
|
||||
|
||||
OldStreamsCleaner.CleanupBatchException failure = assertThrows(
|
||||
OldStreamsCleaner.CleanupBatchException.class,
|
||||
() -> cleaner.cleanupBatch(1_000, Long.MAX_VALUE, policy)
|
||||
);
|
||||
|
||||
assertEquals(1, countWhere("stream", "id = 1 AND cleanup_pending = false"));
|
||||
assertEquals(1, countWhere("packet", "id = 11"));
|
||||
assertEquals(1, countWhere("found_pattern", "id = 21"));
|
||||
assertEquals(1, countWhere("stream_found_patterns", "stream_id = 1 AND pattern_id = 1"));
|
||||
|
||||
long retryAfter = System.currentTimeMillis() + 60_000;
|
||||
cleaner.deferCleanup(failure.streamIds(), Instant.ofEpochMilli(retryAfter));
|
||||
assertEquals(1, countWhere(
|
||||
"stream",
|
||||
"id = 1 AND cleanup_pending = false AND cleanup_attempts = 1 AND cleanup_retry_after = " + retryAfter
|
||||
));
|
||||
|
||||
OldStreamsCleaner.CleanupBatchResult deferred = cleaner.cleanupBatch(
|
||||
1_000,
|
||||
Long.MAX_VALUE,
|
||||
policy
|
||||
);
|
||||
assertEquals(OldStreamsCleaner.CleanupBatchResult.empty(), deferred);
|
||||
|
||||
jdbc.execute("DROP TRIGGER reject_packet_cleanup ON packet");
|
||||
jdbc.execute("DROP FUNCTION reject_packet_cleanup()");
|
||||
jdbc.update("UPDATE stream SET cleanup_retry_after = 0 WHERE id = 1");
|
||||
|
||||
OldStreamsCleaner.CleanupBatchResult resumed = cleaner.cleanupBatch(
|
||||
1_000,
|
||||
Long.MAX_VALUE,
|
||||
policy
|
||||
);
|
||||
assertTrue(resumed.hasWork());
|
||||
assertEquals(0, countWhere("stream", "id = 1"));
|
||||
assertEquals(0, countWhere("packet", "stream_id = 1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rebuildsAnIndexWithAnUnexpectedDefinition() {
|
||||
jdbc.execute("DROP INDEX CONCURRENTLY stream_cleanup_index");
|
||||
jdbc.execute("CREATE INDEX CONCURRENTLY stream_cleanup_index ON stream (id)");
|
||||
|
||||
databaseInitializer.initializeIndexes();
|
||||
|
||||
String definition = indexDefinition("stream_cleanup_index");
|
||||
assertTrue(definition.contains("(end_timestamp, id)"));
|
||||
assertTrue(definition.contains("WHERE ((favorite = false) AND (cleanup_pending = false))"));
|
||||
assertTrue(indexIsValid("stream_cleanup_index"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rebuildsAnInvalidInterruptedConcurrentIndex() {
|
||||
jdbc.execute("DROP INDEX CONCURRENTLY stream_cleanup_index");
|
||||
assertThrows(
|
||||
DataAccessException.class,
|
||||
() -> jdbc.execute("CREATE UNIQUE INDEX CONCURRENTLY stream_cleanup_index ON stream (favorite)")
|
||||
);
|
||||
assertEquals(1, countWhere("pg_indexes", "indexname = 'stream_cleanup_index'"));
|
||||
assertTrue(!indexIsValid("stream_cleanup_index"));
|
||||
|
||||
databaseInitializer.initializeIndexes();
|
||||
|
||||
assertTrue(indexIsValid("stream_cleanup_index"));
|
||||
assertTrue(indexDefinition("stream_cleanup_index").contains("(end_timestamp, id)"));
|
||||
}
|
||||
|
||||
private void insertStream(long id, long endTimestamp, boolean favorite) {
|
||||
jdbc.update("""
|
||||
INSERT INTO stream
|
||||
(id, service_id, protocol, start_timestamp, end_timestamp, favorite, ttl,
|
||||
size_bytes, packets_count)
|
||||
VALUES (?, 80, 0, 0, ?, ?, 64, 1, 1)
|
||||
""", id, endTimestamp, favorite);
|
||||
}
|
||||
|
||||
private void insertPacket(long id, long streamId) {
|
||||
jdbc.update("""
|
||||
INSERT INTO packet
|
||||
(id, stream_id, timestamp, incoming, http_processed, web_socket_parsed,
|
||||
tls_decrypted, has_http_body, content)
|
||||
VALUES (?, ?, 0, true, false, false, false, false, decode('00', 'hex'))
|
||||
""", id, streamId);
|
||||
}
|
||||
|
||||
private int count(String table) {
|
||||
return countWhere(table, "true");
|
||||
}
|
||||
|
||||
private int countWhere(String table, String condition) {
|
||||
return jdbc.queryForObject("SELECT count(*) FROM " + table + " WHERE " + condition, Integer.class);
|
||||
}
|
||||
|
||||
private boolean indexIsValid(String indexName) {
|
||||
return Boolean.TRUE.equals(jdbc.queryForObject("""
|
||||
SELECT i.indisvalid AND i.indisready
|
||||
FROM pg_index i
|
||||
JOIN pg_class index_class ON index_class.oid = i.indexrelid
|
||||
WHERE index_class.relname = ?
|
||||
""", Boolean.class, indexName));
|
||||
}
|
||||
|
||||
private String indexDefinition(String indexName) {
|
||||
return jdbc.queryForObject(
|
||||
"SELECT indexdef FROM pg_indexes WHERE indexname = ?",
|
||||
String.class,
|
||||
indexName
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,687 @@
|
||||
package ru.serega6531.packmate.service;
|
||||
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.Timeout;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase;
|
||||
import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.test.context.DynamicPropertyRegistry;
|
||||
import org.springframework.test.context.DynamicPropertySource;
|
||||
import org.springframework.transaction.annotation.Propagation;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.testcontainers.containers.PostgreSQLContainer;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
import org.testcontainers.junit.jupiter.Testcontainers;
|
||||
import org.testcontainers.utility.DockerImageName;
|
||||
import ru.serega6531.packmate.configuration.CleanupDatabaseInitializer;
|
||||
import ru.serega6531.packmate.properties.PackmateProperties;
|
||||
import ru.serega6531.packmate.tasks.OldStreamsCleanupTask;
|
||||
import ru.serega6531.packmate.tasks.OldStreamsCleanupTask.CleanupRunResult;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.SQLException;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertAll;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@DataJpaTest(showSql = false)
|
||||
@Import({OldStreamsCleaner.class, CleanupDatabaseInitializer.class})
|
||||
@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE)
|
||||
@Transactional(propagation = Propagation.NOT_SUPPORTED)
|
||||
@Testcontainers
|
||||
@Tag("stress")
|
||||
class OldStreamsCleanerStressTest {
|
||||
|
||||
private static final long NEW_STREAM_ID_BASE = 10_000_000;
|
||||
private static final long OVERSIZE_STREAM_ID = NEW_STREAM_ID_BASE - 1;
|
||||
private static final long OVERSIZE_PACKET_ID_BASE = 8_000_000_000L;
|
||||
private static final int FAVORITE_EVERY = 100;
|
||||
|
||||
@Container
|
||||
private static final PostgreSQLContainer<?> POSTGRES = new PostgreSQLContainer<>(
|
||||
DockerImageName.parse("postgres:15.2")
|
||||
)
|
||||
.withDatabaseName("packmate")
|
||||
.withUsername("packmate")
|
||||
.withPassword("test")
|
||||
.withCommand(
|
||||
"postgres",
|
||||
"-c", "autovacuum_naptime=2s",
|
||||
"-c", "autovacuum_vacuum_threshold=1000",
|
||||
"-c", "autovacuum_vacuum_scale_factor=0.02",
|
||||
"-c", "autovacuum_analyze_scale_factor=0.05",
|
||||
"-c", "autovacuum_vacuum_cost_limit=1000"
|
||||
);
|
||||
|
||||
@DynamicPropertySource
|
||||
static void configureDatasource(DynamicPropertyRegistry registry) {
|
||||
registry.add("spring.datasource.url", POSTGRES::getJdbcUrl);
|
||||
registry.add("spring.datasource.username", POSTGRES::getUsername);
|
||||
registry.add("spring.datasource.password", POSTGRES::getPassword);
|
||||
registry.add("spring.jpa.hibernate.ddl-auto", () -> "create");
|
||||
}
|
||||
|
||||
@Autowired
|
||||
private JdbcTemplate jdbc;
|
||||
|
||||
@Autowired
|
||||
private DataSource dataSource;
|
||||
|
||||
@Autowired
|
||||
private OldStreamsCleaner cleaner;
|
||||
|
||||
@Test
|
||||
@Timeout(value = 10, unit = TimeUnit.MINUTES)
|
||||
void keepsIngestResponsiveWhileDeletingACompetitionSizedBacklog() throws Exception {
|
||||
StressConfig config = StressConfig.fromEnvironment();
|
||||
config.validate();
|
||||
prepareBacklog(config);
|
||||
|
||||
long expectedFavoriteStreams = config.oldStreams() / FAVORITE_EVERY;
|
||||
long expectedDeletedStreams = config.oldStreams() - expectedFavoriteStreams + 1;
|
||||
long expectedDeletedPackets = (expectedDeletedStreams - 1) * config.packetsPerStream()
|
||||
+ config.oversizePackets();
|
||||
long autovacuumRunsBefore = autovacuumRuns();
|
||||
long databaseBytesBefore = databaseSize();
|
||||
String walStart = currentWalLsn();
|
||||
|
||||
AtomicBoolean stopWriters = new AtomicBoolean(false);
|
||||
AtomicLong nextStreamId = new AtomicLong(NEW_STREAM_ID_BASE);
|
||||
AtomicLong ingestedStreams = new AtomicLong();
|
||||
AtomicReference<Throwable> writerFailure = new AtomicReference<>();
|
||||
ConcurrentLinkedQueue<Long> ingestLatencyNanos = new ConcurrentLinkedQueue<>();
|
||||
ExecutorService writers = Executors.newFixedThreadPool(config.writerThreads());
|
||||
CountDownLatch writersStarted = new CountDownLatch(config.writerThreads());
|
||||
List<Future<?>> writerFutures = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < config.writerThreads(); i++) {
|
||||
writerFutures.add(writers.submit(() -> runWriter(
|
||||
config,
|
||||
stopWriters,
|
||||
nextStreamId,
|
||||
ingestedStreams,
|
||||
ingestLatencyNanos,
|
||||
writerFailure,
|
||||
writersStarted
|
||||
)));
|
||||
}
|
||||
|
||||
assertTrue(writersStarted.await(30, TimeUnit.SECONDS), "ingest writers did not start");
|
||||
|
||||
PackmateProperties.CleanupProperties cleanupProperties = new PackmateProperties.CleanupProperties(
|
||||
true,
|
||||
15,
|
||||
config.cleanupIntervalSeconds(),
|
||||
config.cleanupMaxRunSeconds(),
|
||||
0,
|
||||
config.maxStreamsPerBatch(),
|
||||
config.maxBatchesPerRun(),
|
||||
Math.toIntExact(config.maxPacketsPerBatch()),
|
||||
Math.toIntExact(config.maxPacketsPerBatch()),
|
||||
config.maxBytesPerBatch(),
|
||||
5,
|
||||
1
|
||||
);
|
||||
OldStreamsCleanupTask cleanupTask = new OldStreamsCleanupTask(
|
||||
cleaner,
|
||||
new PackmateProperties(null, null, null, null, null, null, null, cleanupProperties, true),
|
||||
org.mockito.Mockito.mock(SubscriptionService.class)
|
||||
);
|
||||
|
||||
List<Long> cleanupBatchNanos = new ArrayList<>();
|
||||
int cleanupBatches = 0;
|
||||
long deletedStreams = 0;
|
||||
long deletedPackets = 0;
|
||||
long deletedMatches = 0;
|
||||
long cleanupStarted = System.nanoTime();
|
||||
|
||||
try {
|
||||
while (deletedStreams < expectedDeletedStreams) {
|
||||
Throwable failure = writerFailure.get();
|
||||
if (failure != null) {
|
||||
throw new AssertionError("concurrent ingest failed", failure);
|
||||
}
|
||||
|
||||
CleanupRunResult result = cleanupTask.runCleanup();
|
||||
cleanupBatchNanos.add(result.maxBatchNanos());
|
||||
cleanupBatches += result.batches();
|
||||
|
||||
if (result.failed() || (result.streams() == 0 && result.packets() == 0)) {
|
||||
throw new AssertionError(
|
||||
"cleaner stopped after " + deletedStreams + " of " + expectedDeletedStreams + " streams"
|
||||
);
|
||||
}
|
||||
|
||||
deletedStreams += result.streams();
|
||||
deletedPackets += result.packets();
|
||||
deletedMatches += result.foundPatterns();
|
||||
|
||||
if (deletedStreams < expectedDeletedStreams) {
|
||||
Thread.sleep(TimeUnit.SECONDS.toMillis(config.cleanupIntervalSeconds()));
|
||||
}
|
||||
}
|
||||
|
||||
CleanupRunResult idleRun = cleanupTask.runCleanup();
|
||||
assertEquals(0, idleRun.streams(), "cleanup must stop when only favorite and fresh streams remain");
|
||||
assertEquals(0, idleRun.packets(), "cleanup must stop when only favorite and fresh packets remain");
|
||||
} finally {
|
||||
stopWriters.set(true);
|
||||
writers.shutdown();
|
||||
writers.awaitTermination(30, TimeUnit.SECONDS);
|
||||
for (Future<?> future : writerFutures) {
|
||||
future.get(30, TimeUnit.SECONDS);
|
||||
}
|
||||
}
|
||||
|
||||
long cleanupElapsedNanos = System.nanoTime() - cleanupStarted;
|
||||
Throwable failure = writerFailure.get();
|
||||
long ingested = ingestedStreams.get();
|
||||
List<Long> ingestLatencies = new ArrayList<>(ingestLatencyNanos);
|
||||
double ingestP99Millis = percentileMillis(ingestLatencies, 0.99);
|
||||
double ingestMaxMillis = nanosToMillis(ingestLatencies.stream().mapToLong(Long::longValue).max().orElse(0));
|
||||
double cleanupMaxBatchMillis = nanosToMillis(cleanupBatchNanos.stream().mapToLong(Long::longValue).max().orElse(0));
|
||||
double cleanupSeconds = cleanupElapsedNanos / 1_000_000_000.0;
|
||||
double cleanupPacketsPerSecond = deletedPackets / cleanupSeconds;
|
||||
long walBytes = walBytesSince(walStart);
|
||||
long databaseBytesAfter = databaseSize();
|
||||
|
||||
long deadTuplesBeforeVacuum = estimatedDeadTuples();
|
||||
VacuumResult vacuum = waitForAutovacuum(
|
||||
autovacuumRunsBefore,
|
||||
config.maxDeadTuplesAfterVacuum(),
|
||||
Duration.ofSeconds(config.autovacuumWaitSeconds())
|
||||
);
|
||||
|
||||
long remainingOldNonFavorites = countWhere(
|
||||
"stream", "id < " + NEW_STREAM_ID_BASE + " AND favorite = false"
|
||||
);
|
||||
long remainingFavoriteStreams = countWhere(
|
||||
"stream", "id < " + NEW_STREAM_ID_BASE + " AND favorite = true"
|
||||
);
|
||||
long remainingFavoritePackets = jdbc.queryForObject("""
|
||||
SELECT count(*)
|
||||
FROM packet p
|
||||
JOIN stream s ON s.id = p.stream_id
|
||||
WHERE s.id < ? AND s.favorite = true
|
||||
""", Long.class, NEW_STREAM_ID_BASE);
|
||||
long remainingNewStreams = countWhere("stream", "id >= " + NEW_STREAM_ID_BASE);
|
||||
long remainingNewPackets = jdbc.queryForObject("""
|
||||
SELECT count(*)
|
||||
FROM packet p
|
||||
JOIN stream s ON s.id = p.stream_id
|
||||
WHERE s.id >= ?
|
||||
""", Long.class, NEW_STREAM_ID_BASE);
|
||||
long orphanPackets = jdbc.queryForObject("""
|
||||
SELECT count(*) FROM packet p
|
||||
LEFT JOIN stream s ON s.id = p.stream_id
|
||||
WHERE s.id IS NULL
|
||||
""", Long.class);
|
||||
long orphanMatches = jdbc.queryForObject("""
|
||||
SELECT count(*) FROM found_pattern fp
|
||||
LEFT JOIN packet p ON p.id = fp.packet_id
|
||||
WHERE p.id IS NULL
|
||||
""", Long.class);
|
||||
|
||||
StressResult result = new StressResult(
|
||||
config,
|
||||
deletedStreams,
|
||||
deletedPackets,
|
||||
deletedMatches,
|
||||
cleanupBatches,
|
||||
cleanupSeconds,
|
||||
cleanupPacketsPerSecond,
|
||||
cleanupMaxBatchMillis,
|
||||
ingested,
|
||||
ingestP99Millis,
|
||||
ingestMaxMillis,
|
||||
walBytes,
|
||||
databaseBytesBefore,
|
||||
databaseBytesAfter,
|
||||
deadTuplesBeforeVacuum,
|
||||
vacuum.deadTuples(),
|
||||
vacuum.runs()
|
||||
);
|
||||
writeReport(result);
|
||||
System.out.println(result.summary());
|
||||
|
||||
long actualDeletedStreams = deletedStreams;
|
||||
long actualDeletedPackets = deletedPackets;
|
||||
assertAll(
|
||||
() -> assertNull(failure, "concurrent ingest must not fail"),
|
||||
() -> assertEquals(expectedDeletedStreams, actualDeletedStreams, "all old non-favorite streams"),
|
||||
() -> assertEquals(expectedDeletedPackets, actualDeletedPackets, "all packets of deleted streams"),
|
||||
() -> assertEquals(0, remainingOldNonFavorites, "old non-favorite streams"),
|
||||
() -> assertEquals(expectedFavoriteStreams, remainingFavoriteStreams, "favorite streams"),
|
||||
() -> assertEquals(
|
||||
expectedFavoriteStreams * config.packetsPerStream(),
|
||||
remainingFavoritePackets,
|
||||
"packets of favorite streams"
|
||||
),
|
||||
() -> assertEquals(ingested, remainingNewStreams, "concurrently ingested streams"),
|
||||
() -> assertEquals(
|
||||
ingested * config.packetsPerStream(),
|
||||
remainingNewPackets,
|
||||
"packets of concurrently ingested streams"
|
||||
),
|
||||
() -> assertEquals(0, orphanPackets, "orphan packets"),
|
||||
() -> assertEquals(0, orphanMatches, "orphan matches"),
|
||||
() -> assertEquals(1, countWhere("pg_indexes", "indexname = 'stream_cleanup_index'")),
|
||||
() -> assertEquals(1, countWhere("pg_indexes", "indexname = 'stream_cleanup_pending_index'")),
|
||||
() -> assertEquals(1, countWhere("pg_indexes", "indexname = 'packet_cleanup_stream_id_index'")),
|
||||
() -> assertTrue(ingested > 0, "stress test must overlap cleanup with ingest"),
|
||||
() -> assertTrue(
|
||||
vacuum.runs() > autovacuumRunsBefore,
|
||||
"autovacuum must process the deleted packet rows"
|
||||
),
|
||||
() -> assertTrue(
|
||||
vacuum.deadTuples() <= config.maxDeadTuplesAfterVacuum(),
|
||||
"dead tuples after autovacuum: " + vacuum.deadTuples()
|
||||
),
|
||||
() -> assertTrue(
|
||||
ingestP99Millis <= config.maxIngestP99Millis(),
|
||||
String.format(Locale.ROOT, "ingest p99 %.2f ms exceeds %.2f ms",
|
||||
ingestP99Millis, config.maxIngestP99Millis())
|
||||
),
|
||||
() -> assertTrue(
|
||||
cleanupMaxBatchMillis <= config.maxCleanupBatchMillis(),
|
||||
String.format(Locale.ROOT, "cleanup batch %.2f ms exceeds %.2f ms",
|
||||
cleanupMaxBatchMillis, config.maxCleanupBatchMillis())
|
||||
),
|
||||
() -> assertTrue(
|
||||
cleanupPacketsPerSecond >= config.minCleanupPacketsPerSecond(),
|
||||
String.format(Locale.ROOT, "cleanup throughput %.0f packets/s is below %.0f packets/s",
|
||||
cleanupPacketsPerSecond, config.minCleanupPacketsPerSecond())
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
private void prepareBacklog(StressConfig config) {
|
||||
jdbc.execute("TRUNCATE TABLE stream_found_patterns, found_pattern, packet, stream, pattern CASCADE");
|
||||
jdbc.update("""
|
||||
INSERT INTO pattern
|
||||
(id, enabled, deleted, name, value, color, search_type, direction_type,
|
||||
action_type, search_start_timestamp, search_target)
|
||||
VALUES (1, true, false, 'stress', 'stress', '#fff', 0, 0, 0, 0, 'CONTENT')
|
||||
""");
|
||||
|
||||
jdbc.update("""
|
||||
INSERT INTO stream
|
||||
(id, service_id, protocol, start_timestamp, end_timestamp, favorite, ttl,
|
||||
size_bytes, packets_count)
|
||||
SELECT id, 80, 0, 0, 0, id % ? = 0, 64, ?, ?
|
||||
FROM generate_series(1, ?) AS id
|
||||
""", FAVORITE_EVERY,
|
||||
config.packetsPerStream() * config.payloadBytes(),
|
||||
config.packetsPerStream(),
|
||||
config.oldStreams());
|
||||
|
||||
jdbc.update("""
|
||||
INSERT INTO packet
|
||||
(id, stream_id, timestamp, incoming, http_processed, web_socket_parsed,
|
||||
tls_decrypted, has_http_body, content)
|
||||
SELECT ((s.id - 1) * CAST(? AS bigint)) + packet_number,
|
||||
s.id, 0, packet_number % 2 = 0, false, false, false, false,
|
||||
decode(repeat(md5(s.id::text || ':' || packet_number::text), ?), 'hex')
|
||||
FROM stream s
|
||||
CROSS JOIN generate_series(1, ?) AS packet_number
|
||||
WHERE s.id <= ?
|
||||
""", config.packetsPerStream(), config.payloadBytes() / 16,
|
||||
config.packetsPerStream(), config.oldStreams());
|
||||
|
||||
jdbc.update("""
|
||||
INSERT INTO found_pattern
|
||||
(id, packet_id, pattern_id, search_target, start_position, end_position)
|
||||
SELECT id, id, 1, 'CONTENT', 0, 1
|
||||
FROM packet
|
||||
WHERE stream_id <= ? AND id % 10 = 0
|
||||
""", config.oldStreams());
|
||||
jdbc.update("""
|
||||
INSERT INTO stream_found_patterns (stream_id, pattern_id)
|
||||
SELECT id, 1 FROM stream WHERE id <= ? AND id % 10 = 0
|
||||
""", config.oldStreams());
|
||||
|
||||
// Deliberately lying metadata verifies that the deletion chunk is bounded by actual packet rows/bytes.
|
||||
jdbc.update("""
|
||||
INSERT INTO stream
|
||||
(id, service_id, protocol, start_timestamp, end_timestamp, favorite, ttl,
|
||||
size_bytes, packets_count)
|
||||
VALUES (?, 80, 0, 0, -1, false, 64, 1, 1)
|
||||
""", OVERSIZE_STREAM_ID);
|
||||
jdbc.update("""
|
||||
INSERT INTO packet
|
||||
(id, stream_id, timestamp, incoming, http_processed, web_socket_parsed,
|
||||
tls_decrypted, has_http_body, content)
|
||||
SELECT ? + packet_number, ?, 0, true, false, false, false, false,
|
||||
decode(repeat(md5(packet_number::text), ?), 'hex')
|
||||
FROM generate_series(1, ?) AS packet_number
|
||||
""", OVERSIZE_PACKET_ID_BASE, OVERSIZE_STREAM_ID,
|
||||
config.oversizePayloadBytes() / 16, config.oversizePackets());
|
||||
|
||||
jdbc.execute("ANALYZE stream");
|
||||
jdbc.execute("ANALYZE packet");
|
||||
jdbc.execute("ANALYZE found_pattern");
|
||||
}
|
||||
|
||||
private void runWriter(StressConfig config,
|
||||
AtomicBoolean stop,
|
||||
AtomicLong nextStreamId,
|
||||
AtomicLong ingestedStreams,
|
||||
ConcurrentLinkedQueue<Long> latencies,
|
||||
AtomicReference<Throwable> failure,
|
||||
CountDownLatch started) {
|
||||
byte[] content = new byte[config.payloadBytes()];
|
||||
for (int i = 0; i < content.length; i++) {
|
||||
content[i] = (byte) (i * 31 + 17);
|
||||
}
|
||||
|
||||
try (Connection connection = dataSource.getConnection();
|
||||
PreparedStatement insertStream = connection.prepareStatement("""
|
||||
INSERT INTO stream
|
||||
(id, service_id, protocol, start_timestamp, end_timestamp, favorite, ttl,
|
||||
persisted_at, size_bytes, packets_count)
|
||||
VALUES (?, 80, 0,
|
||||
CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS bigint),
|
||||
CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS bigint),
|
||||
false, 64,
|
||||
CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS bigint),
|
||||
?, ?)
|
||||
""");
|
||||
PreparedStatement insertPacket = connection.prepareStatement("""
|
||||
INSERT INTO packet
|
||||
(id, stream_id, timestamp, incoming, http_processed, web_socket_parsed,
|
||||
tls_decrypted, has_http_body, content)
|
||||
VALUES (?, ?, CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS bigint),
|
||||
true, false, false, false, false, ?)
|
||||
""")) {
|
||||
connection.setAutoCommit(false);
|
||||
started.countDown();
|
||||
|
||||
while (!stop.get()) {
|
||||
long streamId = nextStreamId.getAndIncrement();
|
||||
long startedAt = System.nanoTime();
|
||||
|
||||
insertStream.setLong(1, streamId);
|
||||
insertStream.setInt(2, config.packetsPerStream() * config.payloadBytes());
|
||||
insertStream.setInt(3, config.packetsPerStream());
|
||||
insertStream.executeUpdate();
|
||||
|
||||
for (int packet = 1; packet <= config.packetsPerStream(); packet++) {
|
||||
insertPacket.setLong(1, streamId * 1_000 + packet);
|
||||
insertPacket.setLong(2, streamId);
|
||||
insertPacket.setBytes(3, content);
|
||||
insertPacket.addBatch();
|
||||
}
|
||||
insertPacket.executeBatch();
|
||||
connection.commit();
|
||||
|
||||
latencies.add(System.nanoTime() - startedAt);
|
||||
ingestedStreams.incrementAndGet();
|
||||
|
||||
if (config.writerPauseMillis() > 0) {
|
||||
Thread.sleep(config.writerPauseMillis());
|
||||
}
|
||||
}
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
} catch (Throwable throwable) {
|
||||
failure.compareAndSet(null, throwable);
|
||||
stop.set(true);
|
||||
}
|
||||
}
|
||||
|
||||
private long databaseSize() {
|
||||
return jdbc.queryForObject("SELECT pg_database_size(current_database())", Long.class);
|
||||
}
|
||||
|
||||
private String currentWalLsn() {
|
||||
return jdbc.queryForObject("SELECT pg_current_wal_lsn()::text", String.class);
|
||||
}
|
||||
|
||||
private long walBytesSince(String walStart) {
|
||||
Number result = jdbc.queryForObject(
|
||||
"SELECT pg_wal_lsn_diff(pg_current_wal_lsn(), CAST(? AS pg_lsn))",
|
||||
Number.class,
|
||||
walStart
|
||||
);
|
||||
return result.longValue();
|
||||
}
|
||||
|
||||
private long estimatedDeadTuples() {
|
||||
Long result = jdbc.queryForObject("""
|
||||
SELECT COALESCE(sum(n_dead_tup), 0)
|
||||
FROM pg_stat_user_tables
|
||||
WHERE relname IN ('stream', 'packet', 'found_pattern', 'stream_found_patterns')
|
||||
""", Long.class);
|
||||
return result == null ? 0 : result;
|
||||
}
|
||||
|
||||
private long autovacuumRuns() {
|
||||
Long result = jdbc.queryForObject("""
|
||||
SELECT COALESCE(sum(autovacuum_count), 0)
|
||||
FROM pg_stat_user_tables
|
||||
WHERE relname IN ('stream', 'packet', 'found_pattern', 'stream_found_patterns')
|
||||
""", Long.class);
|
||||
return result == null ? 0 : result;
|
||||
}
|
||||
|
||||
private VacuumResult waitForAutovacuum(long initialRuns, long maxDeadTuples, Duration timeout)
|
||||
throws InterruptedException {
|
||||
long deadline = System.nanoTime() + timeout.toNanos();
|
||||
long runs = initialRuns;
|
||||
long deadTuples = Long.MAX_VALUE;
|
||||
|
||||
while (System.nanoTime() < deadline) {
|
||||
jdbc.execute("SELECT pg_stat_clear_snapshot()");
|
||||
runs = autovacuumRuns();
|
||||
deadTuples = estimatedDeadTuples();
|
||||
if (runs > initialRuns && deadTuples <= maxDeadTuples) {
|
||||
return new VacuumResult(runs, deadTuples);
|
||||
}
|
||||
Thread.sleep(1_000);
|
||||
}
|
||||
|
||||
return new VacuumResult(runs, deadTuples);
|
||||
}
|
||||
|
||||
private long countWhere(String table, String condition) {
|
||||
return jdbc.queryForObject("SELECT count(*) FROM " + table + " WHERE " + condition, Long.class);
|
||||
}
|
||||
|
||||
private double percentileMillis(List<Long> values, double percentile) {
|
||||
if (values.isEmpty()) {
|
||||
return Double.POSITIVE_INFINITY;
|
||||
}
|
||||
Collections.sort(values);
|
||||
int index = Math.max(0, (int) Math.ceil(values.size() * percentile) - 1);
|
||||
return nanosToMillis(values.get(index));
|
||||
}
|
||||
|
||||
private double nanosToMillis(long nanos) {
|
||||
return nanos / 1_000_000.0;
|
||||
}
|
||||
|
||||
private void writeReport(StressResult result) throws IOException {
|
||||
Path report = Path.of("build", "reports", "cleanup-stress.json");
|
||||
Files.createDirectories(report.getParent());
|
||||
Files.writeString(report, result.json());
|
||||
}
|
||||
|
||||
private record StressConfig(
|
||||
int oldStreams,
|
||||
int packetsPerStream,
|
||||
int payloadBytes,
|
||||
int oversizePackets,
|
||||
int oversizePayloadBytes,
|
||||
int writerThreads,
|
||||
int writerPauseMillis,
|
||||
int maxStreamsPerBatch,
|
||||
int maxBatchesPerRun,
|
||||
int cleanupIntervalSeconds,
|
||||
int cleanupMaxRunSeconds,
|
||||
long maxPacketsPerBatch,
|
||||
long maxBytesPerBatch,
|
||||
double maxIngestP99Millis,
|
||||
double maxCleanupBatchMillis,
|
||||
double minCleanupPacketsPerSecond,
|
||||
long maxDeadTuplesAfterVacuum,
|
||||
int autovacuumWaitSeconds
|
||||
) {
|
||||
static StressConfig fromEnvironment() {
|
||||
return new StressConfig(
|
||||
envInt("PACKMATE_STRESS_STREAMS", 30_000),
|
||||
envInt("PACKMATE_STRESS_PACKETS_PER_STREAM", 30),
|
||||
envInt("PACKMATE_STRESS_PAYLOAD_BYTES", 256),
|
||||
envInt("PACKMATE_STRESS_OVERSIZE_PACKETS", 25_000),
|
||||
envInt("PACKMATE_STRESS_OVERSIZE_PAYLOAD_BYTES", 4_096),
|
||||
envInt("PACKMATE_STRESS_WRITERS", 2),
|
||||
envInt("PACKMATE_STRESS_WRITER_PAUSE_MS", 5),
|
||||
envInt("PACKMATE_STRESS_BATCH_STREAMS", 500),
|
||||
envInt("PACKMATE_STRESS_MAX_BATCHES_PER_RUN", 20),
|
||||
envInt("PACKMATE_STRESS_CLEANUP_INTERVAL_SECONDS", 10),
|
||||
envInt("PACKMATE_STRESS_CLEANUP_MAX_RUN_SECONDS", 5),
|
||||
envLong("PACKMATE_STRESS_BATCH_PACKETS", 20_000),
|
||||
envLong("PACKMATE_STRESS_BATCH_BYTES", 67_108_864),
|
||||
envDouble("PACKMATE_STRESS_MAX_INGEST_P99_MS", 500),
|
||||
envDouble("PACKMATE_STRESS_MAX_BATCH_MS", 5_000),
|
||||
envDouble("PACKMATE_STRESS_MIN_CLEANUP_PACKETS_PER_SECOND", 5_000),
|
||||
envLong("PACKMATE_STRESS_MAX_DEAD_TUPLES", 50_000),
|
||||
envInt("PACKMATE_STRESS_AUTOVACUUM_WAIT_SECONDS", 90)
|
||||
);
|
||||
}
|
||||
|
||||
void validate() {
|
||||
if (oldStreams <= 0 || oldStreams >= OVERSIZE_STREAM_ID || packetsPerStream <= 0
|
||||
|| payloadBytes <= 0 || payloadBytes % 16 != 0 || oversizePackets <= 0
|
||||
|| oversizePayloadBytes <= 0 || oversizePayloadBytes % 16 != 0) {
|
||||
throw new IllegalArgumentException(
|
||||
"stress sizes must be positive and PACKMATE_STRESS_PAYLOAD_BYTES must be divisible by 16"
|
||||
);
|
||||
}
|
||||
if (packetsPerStream >= 1_000 || writerThreads <= 0 || writerPauseMillis < 0
|
||||
|| maxBatchesPerRun <= 0 || cleanupIntervalSeconds <= 0 || cleanupMaxRunSeconds <= 0) {
|
||||
throw new IllegalArgumentException("invalid writer configuration");
|
||||
}
|
||||
}
|
||||
|
||||
private static int envInt(String name, int defaultValue) {
|
||||
String value = System.getenv(name);
|
||||
return value == null ? defaultValue : Integer.parseInt(value);
|
||||
}
|
||||
|
||||
private static long envLong(String name, long defaultValue) {
|
||||
String value = System.getenv(name);
|
||||
return value == null ? defaultValue : Long.parseLong(value);
|
||||
}
|
||||
|
||||
private static double envDouble(String name, double defaultValue) {
|
||||
String value = System.getenv(name);
|
||||
return value == null ? defaultValue : Double.parseDouble(value);
|
||||
}
|
||||
}
|
||||
|
||||
private record StressResult(
|
||||
StressConfig config,
|
||||
long deletedStreams,
|
||||
long deletedPackets,
|
||||
long deletedMatches,
|
||||
int cleanupBatches,
|
||||
double cleanupSeconds,
|
||||
double cleanupPacketsPerSecond,
|
||||
double cleanupMaxBatchMillis,
|
||||
long ingestedStreams,
|
||||
double ingestP99Millis,
|
||||
double ingestMaxMillis,
|
||||
long walBytes,
|
||||
long databaseBytesBefore,
|
||||
long databaseBytesAfter,
|
||||
long deadTuplesBeforeVacuum,
|
||||
long deadTuplesAfterVacuum,
|
||||
long autovacuumRuns
|
||||
) {
|
||||
String summary() {
|
||||
return String.format(
|
||||
Locale.ROOT,
|
||||
"Cleanup stress: deleted %,d streams / %,d packets in %.2fs (%,.0f packets/s), "
|
||||
+ "%d batches, max batch %.2fms; concurrent ingest %,d streams, p99 %.2fms, "
|
||||
+ "max %.2fms; WAL %.2fMiB, DB %.2fMiB -> %.2fMiB, "
|
||||
+ "dead tuples %,d -> %,d after %d autovacuum runs",
|
||||
deletedStreams, deletedPackets, cleanupSeconds, cleanupPacketsPerSecond,
|
||||
cleanupBatches, cleanupMaxBatchMillis, ingestedStreams, ingestP99Millis,
|
||||
ingestMaxMillis, walBytes / 1_048_576.0, databaseBytesBefore / 1_048_576.0,
|
||||
databaseBytesAfter / 1_048_576.0, deadTuplesBeforeVacuum,
|
||||
deadTuplesAfterVacuum, autovacuumRuns
|
||||
);
|
||||
}
|
||||
|
||||
String json() {
|
||||
return String.format(
|
||||
Locale.ROOT,
|
||||
"""
|
||||
{
|
||||
"generatedAt": "%s",
|
||||
"oldStreams": %d,
|
||||
"packetsPerStream": %d,
|
||||
"payloadBytes": %d,
|
||||
"oversizePackets": %d,
|
||||
"oversizePayloadBytes": %d,
|
||||
"writerThreads": %d,
|
||||
"maxBatchesPerRun": %d,
|
||||
"cleanupIntervalSeconds": %d,
|
||||
"cleanupMaxRunSeconds": %d,
|
||||
"deletedStreams": %d,
|
||||
"deletedPackets": %d,
|
||||
"deletedMatches": %d,
|
||||
"cleanupBatches": %d,
|
||||
"cleanupSeconds": %.3f,
|
||||
"cleanupPacketsPerSecond": %.3f,
|
||||
"cleanupMaxBatchMillis": %.3f,
|
||||
"ingestedStreams": %d,
|
||||
"ingestP99Millis": %.3f,
|
||||
"ingestMaxMillis": %.3f,
|
||||
"walBytes": %d,
|
||||
"databaseBytesBefore": %d,
|
||||
"databaseBytesAfter": %d,
|
||||
"deadTuplesBeforeVacuum": %d,
|
||||
"deadTuplesAfterVacuum": %d,
|
||||
"autovacuumRuns": %d
|
||||
}
|
||||
""",
|
||||
Instant.now(), config.oldStreams(), config.packetsPerStream(), config.payloadBytes(),
|
||||
config.oversizePackets(), config.oversizePayloadBytes(), config.writerThreads(),
|
||||
config.maxBatchesPerRun(), config.cleanupIntervalSeconds(),
|
||||
config.cleanupMaxRunSeconds(), deletedStreams, deletedPackets, deletedMatches, cleanupBatches,
|
||||
cleanupSeconds, cleanupPacketsPerSecond, cleanupMaxBatchMillis, ingestedStreams,
|
||||
ingestP99Millis, ingestMaxMillis, walBytes, databaseBytesBefore, databaseBytesAfter,
|
||||
deadTuplesBeforeVacuum, deadTuplesAfterVacuum, autovacuumRuns
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private record VacuumResult(long runs, long deadTuples) {
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,141 @@
|
||||
package ru.serega6531.packmate.service;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.InOrder;
|
||||
import ru.serega6531.packmate.repository.StreamRepository;
|
||||
import ru.serega6531.packmate.repository.StreamRepository.CleanupCandidate;
|
||||
import ru.serega6531.packmate.repository.StreamRepository.CleanupPacket;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.Mockito.inOrder;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class OldStreamsCleanerTest {
|
||||
|
||||
private final StreamRepository repository = mock(StreamRepository.class);
|
||||
private final OldStreamsCleaner cleaner = new OldStreamsCleaner(repository);
|
||||
private final OldStreamsCleaner.CleanupBatchPolicy policy = new OldStreamsCleaner.CleanupBatchPolicy(
|
||||
50, 1_000, 1_000, 1_000_000, Duration.ofSeconds(30), Duration.ofSeconds(2)
|
||||
);
|
||||
|
||||
@Test
|
||||
void deletesOneBatchInForeignKeyOrder() {
|
||||
List<Long> ids = List.of(10L, 20L);
|
||||
CleanupCandidate first = candidate(10L, 10, 100);
|
||||
CleanupCandidate second = candidate(20L, 20, 200);
|
||||
List<Long> packetIds = List.of(1L, 2L, 3L, 4L, 5L);
|
||||
List<CleanupPacket> cleanupPackets = packetIds.stream()
|
||||
.map(id -> packet(id, 10))
|
||||
.toList();
|
||||
when(repository.findAndLockOldStreamCandidates(eq(123L), eq(456L), anyLong(), eq(50)))
|
||||
.thenReturn(List.of(first, second));
|
||||
when(repository.markCleanupPending(ids)).thenReturn(2);
|
||||
when(repository.deletePatternLinks(ids)).thenReturn(3, 0);
|
||||
when(repository.findPacketsForCleanup(ids, 1_000)).thenReturn(cleanupPackets);
|
||||
when(repository.findFoundPatternsForCleanup(packetIds, 1_000)).thenReturn(packetIds);
|
||||
when(repository.deleteFoundPatterns(packetIds)).thenReturn(4);
|
||||
when(repository.deletePacketsWithoutMatches(packetIds)).thenReturn(5);
|
||||
when(repository.deleteFinishedStreams(ids)).thenReturn(2);
|
||||
|
||||
OldStreamsCleaner.CleanupBatchResult result = cleaner.cleanupBatch(123L, 456L, policy);
|
||||
|
||||
assertEquals(new OldStreamsCleaner.CleanupBatchResult(2, 5, 4, 3), result);
|
||||
InOrder order = inOrder(repository);
|
||||
order.verify(repository).configureLocalTimeouts(anyLong(), anyLong());
|
||||
order.verify(repository).findAndLockPendingStreams(anyLong(), eq(50));
|
||||
order.verify(repository).findAndLockOldStreamCandidates(eq(123L), eq(456L), anyLong(), eq(50));
|
||||
order.verify(repository).markCleanupPending(ids);
|
||||
order.verify(repository).deletePatternLinks(ids);
|
||||
order.verify(repository).findPacketsForCleanup(ids, 1_000);
|
||||
order.verify(repository).findFoundPatternsForCleanup(packetIds, 1_000);
|
||||
order.verify(repository).deleteFoundPatterns(packetIds);
|
||||
order.verify(repository).deletePacketsWithoutMatches(packetIds);
|
||||
order.verify(repository).deletePatternLinks(ids);
|
||||
order.verify(repository).deleteFinishedStreams(ids);
|
||||
order.verify(repository).resetCleanupRetry(ids);
|
||||
}
|
||||
|
||||
@Test
|
||||
void doesNotIssueDeletesForAnEmptyBatch() {
|
||||
when(repository.findAndLockOldStreamCandidates(eq(123L), eq(456L), anyLong(), eq(50)))
|
||||
.thenReturn(List.of());
|
||||
|
||||
assertEquals(OldStreamsCleaner.CleanupBatchResult.empty(), cleaner.cleanupBatch(123L, 456L, policy));
|
||||
verify(repository, never()).deletePatternLinks(anyList());
|
||||
verify(repository, never()).deleteFoundPatterns(anyList());
|
||||
verify(repository, never()).deletePacketsWithoutMatches(anyList());
|
||||
verify(repository, never()).deleteFinishedStreams(anyList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void failsTheBatchWhenNotAllLockedStreamsWereClaimed() {
|
||||
List<Long> ids = List.of(10L, 20L);
|
||||
CleanupCandidate first = candidate(10L, 10, 100);
|
||||
CleanupCandidate second = candidate(20L, 20, 200);
|
||||
when(repository.findAndLockOldStreamCandidates(eq(123L), eq(456L), anyLong(), eq(50)))
|
||||
.thenReturn(List.of(first, second));
|
||||
when(repository.markCleanupPending(ids)).thenReturn(1);
|
||||
|
||||
assertThrows(OldStreamsCleaner.CleanupBatchException.class,
|
||||
() -> cleaner.cleanupBatch(123L, 456L, policy));
|
||||
}
|
||||
|
||||
@Test
|
||||
void limitsABatchByPacketAndByteBudgets() {
|
||||
List<Long> selectedIds = List.of(10L);
|
||||
CleanupCandidate first = candidate(10L, 600, 600_000);
|
||||
CleanupCandidate second = candidate(20L, 600, 600_000);
|
||||
when(repository.findAndLockOldStreamCandidates(eq(123L), eq(456L), anyLong(), eq(50)))
|
||||
.thenReturn(List.of(first, second));
|
||||
when(repository.markCleanupPending(selectedIds)).thenReturn(1);
|
||||
when(repository.deleteFinishedStreams(selectedIds)).thenReturn(1);
|
||||
|
||||
cleaner.cleanupBatch(123L, 456L, policy);
|
||||
|
||||
verify(repository).markCleanupPending(selectedIds);
|
||||
}
|
||||
|
||||
@Test
|
||||
void chunksPacketsByActualStoredBytesWhenStreamMetadataIsWrong() {
|
||||
List<Long> streamIds = List.of(10L);
|
||||
List<Long> selectedPacketIds = List.of(1L);
|
||||
CleanupCandidate candidate = candidate(10L, 1, 1);
|
||||
List<CleanupPacket> cleanupPackets = List.of(packet(1L, 600_000), packet(2L, 600_000));
|
||||
when(repository.findAndLockOldStreamCandidates(eq(123L), eq(456L), anyLong(), eq(50)))
|
||||
.thenReturn(List.of(candidate));
|
||||
when(repository.markCleanupPending(streamIds)).thenReturn(1);
|
||||
when(repository.findPacketsForCleanup(streamIds, 1_000)).thenReturn(cleanupPackets);
|
||||
when(repository.deletePacketsWithoutMatches(selectedPacketIds)).thenReturn(1);
|
||||
|
||||
OldStreamsCleaner.CleanupBatchResult result = cleaner.cleanupBatch(123L, 456L, policy);
|
||||
|
||||
assertEquals(1, result.packets());
|
||||
assertEquals(0, result.streams());
|
||||
verify(repository).deletePacketsWithoutMatches(selectedPacketIds);
|
||||
}
|
||||
|
||||
private CleanupCandidate candidate(long id, long packets, long bytes) {
|
||||
CleanupCandidate candidate = mock(CleanupCandidate.class);
|
||||
when(candidate.getId()).thenReturn(id);
|
||||
when(candidate.getPacketsCount()).thenReturn(packets);
|
||||
when(candidate.getSizeBytes()).thenReturn(bytes);
|
||||
return candidate;
|
||||
}
|
||||
|
||||
private CleanupPacket packet(long id, long bytes) {
|
||||
CleanupPacket packet = mock(CleanupPacket.class);
|
||||
when(packet.getId()).thenReturn(id);
|
||||
when(packet.getSizeBytes()).thenReturn(bytes);
|
||||
return packet;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,47 @@
|
||||
package ru.serega6531.packmate.service;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import ru.serega6531.packmate.model.Pattern;
|
||||
import ru.serega6531.packmate.model.enums.PatternSearchTarget;
|
||||
import ru.serega6531.packmate.model.enums.PatternSearchType;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class PatternMatcherConcurrencyTest {
|
||||
|
||||
@Test
|
||||
void substringSearchersDoNotShareMutableStateAcrossWorkers() throws Exception {
|
||||
Pattern pattern = new Pattern();
|
||||
pattern.setId(1);
|
||||
pattern.setValue("PMSTRESS");
|
||||
pattern.setSearchType(PatternSearchType.SUBSTRING);
|
||||
pattern.setSearchTarget(PatternSearchTarget.CONTENT);
|
||||
|
||||
byte[] content = "before-PMSTRESS-middle-PMSTRESS-after".getBytes(StandardCharsets.UTF_8);
|
||||
Callable<Integer> matcher = () -> {
|
||||
int matches = 0;
|
||||
for (int iteration = 0; iteration < 2_000; iteration++) {
|
||||
matches += new PatternMatcher(content, List.of(pattern)).findMatches().size();
|
||||
}
|
||||
return matches;
|
||||
};
|
||||
|
||||
var executor = Executors.newFixedThreadPool(8);
|
||||
try {
|
||||
var futures = executor.invokeAll(java.util.Collections.nCopies(16, matcher));
|
||||
for (var future : futures) {
|
||||
assertEquals(4_000, future.get());
|
||||
}
|
||||
} finally {
|
||||
executor.shutdownNow();
|
||||
assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS));
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,95 @@
|
||||
package ru.serega6531.packmate.tasks;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import ru.serega6531.packmate.properties.PackmateProperties;
|
||||
import ru.serega6531.packmate.service.OldStreamsCleaner;
|
||||
import ru.serega6531.packmate.service.OldStreamsCleaner.CleanupBatchResult;
|
||||
import ru.serega6531.packmate.service.SubscriptionService;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class OldStreamsCleanupTaskTest {
|
||||
|
||||
@Test
|
||||
void stopsAfterAnEmptyBatch() {
|
||||
OldStreamsCleaner cleaner = mock(OldStreamsCleaner.class);
|
||||
when(cleaner.cleanupBatch(anyLong(), anyLong(), any()))
|
||||
.thenReturn(new CleanupBatchResult(10, 100, 5, 2))
|
||||
.thenReturn(CleanupBatchResult.empty());
|
||||
|
||||
task(cleaner, 3).cleanup();
|
||||
|
||||
verify(cleaner, times(2)).cleanupBatch(anyLong(), anyLong(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void stopsAtConfiguredBatchLimit() {
|
||||
OldStreamsCleaner cleaner = mock(OldStreamsCleaner.class);
|
||||
when(cleaner.cleanupBatch(anyLong(), anyLong(), any()))
|
||||
.thenReturn(new CleanupBatchResult(10, 100, 5, 2));
|
||||
|
||||
task(cleaner, 3).cleanup();
|
||||
|
||||
verify(cleaner, times(3)).cleanupBatch(anyLong(), anyLong(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void defersAStreamAfterAFailedBatch() {
|
||||
OldStreamsCleaner cleaner = mock(OldStreamsCleaner.class);
|
||||
List<Long> streamIds = List.of(10L);
|
||||
when(cleaner.cleanupBatch(anyLong(), anyLong(), any()))
|
||||
.thenThrow(new OldStreamsCleaner.CleanupBatchException(
|
||||
streamIds,
|
||||
new RuntimeException("timeout")
|
||||
))
|
||||
.thenReturn(CleanupBatchResult.empty());
|
||||
|
||||
task(cleaner, 3).cleanup();
|
||||
|
||||
verify(cleaner).deferCleanup(org.mockito.ArgumentMatchers.eq(streamIds), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void isolatesAFailedGroupBeforeDeferringAnyStream() {
|
||||
OldStreamsCleaner cleaner = mock(OldStreamsCleaner.class);
|
||||
List<Long> streamIds = List.of(10L, 20L);
|
||||
when(cleaner.cleanupBatch(anyLong(), anyLong(), any()))
|
||||
.thenThrow(new OldStreamsCleaner.CleanupBatchException(
|
||||
streamIds,
|
||||
new RuntimeException("timeout")
|
||||
))
|
||||
.thenReturn(new CleanupBatchResult(1, 10, 0, 0))
|
||||
.thenReturn(CleanupBatchResult.empty());
|
||||
|
||||
task(cleaner, 3).cleanup();
|
||||
|
||||
org.mockito.ArgumentCaptor<OldStreamsCleaner.CleanupBatchPolicy> policies =
|
||||
org.mockito.ArgumentCaptor.forClass(OldStreamsCleaner.CleanupBatchPolicy.class);
|
||||
verify(cleaner, times(3)).cleanupBatch(anyLong(), anyLong(), policies.capture());
|
||||
assertEquals(500, policies.getAllValues().get(0).maxStreams());
|
||||
assertEquals(1, policies.getAllValues().get(1).maxStreams());
|
||||
assertEquals(1, policies.getAllValues().get(2).maxStreams());
|
||||
verify(cleaner, never()).deferCleanup(eq(streamIds), any());
|
||||
}
|
||||
|
||||
private OldStreamsCleanupTask task(OldStreamsCleaner cleaner, int maxBatchesPerRun) {
|
||||
PackmateProperties.CleanupProperties cleanup = new PackmateProperties.CleanupProperties(
|
||||
true, 15, 10, 5, 120, 500, maxBatchesPerRun,
|
||||
20_000, 20_000, 67_108_864, 5, 1
|
||||
);
|
||||
PackmateProperties properties = new PackmateProperties(
|
||||
null, null, null, null, null, null, null, cleanup, true
|
||||
);
|
||||
return new OldStreamsCleanupTask(cleaner, properties, mock(SubscriptionService.class));
|
||||
}
|
||||
}
|
||||
9
src/test/resources/logback-test.xml
Normal file
@ -0,0 +1,9 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<configuration>
|
||||
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
|
||||
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
|
||||
|
||||
<root level="INFO">
|
||||
<appender-ref ref="CONSOLE"/>
|
||||
</root>
|
||||
</configuration>
|
||||
4
stress/network/Dockerfile.tools
Normal file
@ -0,0 +1,4 @@
|
||||
FROM python:3.12-alpine
|
||||
|
||||
WORKDIR /stress
|
||||
COPY tcp_target.py network_load.py api_control.py verify_ledger.py ./
|
||||
427
stress/network/api_control.py
Normal file
@ -0,0 +1,427 @@
|
||||
import argparse
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import struct
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from network_load import PAYLOAD_MARKER, build_payload
|
||||
|
||||
|
||||
BASE_URL = os.getenv("STRESS_API_URL", "http://172.31.250.10:65000/api").rstrip("/")
|
||||
USER = os.getenv("STRESS_API_USER", "stress")
|
||||
PASSWORD = os.getenv("STRESS_API_PASSWORD", "stress")
|
||||
AUTHORIZATION = "Basic " + base64.b64encode(f"{USER}:{PASSWORD}".encode()).decode()
|
||||
EXPECTED_LOCAL_IP = os.getenv("STRESS_TARGET_HOST", "172.31.250.10")
|
||||
|
||||
|
||||
def request(method: str, path: str, body=None, timeout: float = 10):
|
||||
payload = None if body is None else json.dumps(body).encode()
|
||||
headers = {"Authorization": AUTHORIZATION}
|
||||
if payload is not None:
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(BASE_URL + path, data=payload, headers=headers, method=method)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||
data = response.read()
|
||||
return json.loads(data) if data else None
|
||||
|
||||
|
||||
def wait_and_prepare(timeout: int) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
try:
|
||||
request("GET", "/service/")
|
||||
break
|
||||
except (urllib.error.URLError, TimeoutError):
|
||||
if time.monotonic() >= deadline:
|
||||
raise RuntimeError("Packmate API did not become ready")
|
||||
time.sleep(1)
|
||||
|
||||
services = request("GET", "/service/")
|
||||
if not any(service["port"] == 18080 for service in services):
|
||||
request("POST", "/service/", {
|
||||
"port": 18080,
|
||||
"name": "network-stress",
|
||||
"decryptTls": False,
|
||||
"http": False,
|
||||
"urldecodeHttpRequests": False,
|
||||
"mergeAdjacentPackets": False,
|
||||
"parseWebSockets": False,
|
||||
})
|
||||
patterns = request("GET", "/pattern/")
|
||||
if not any(pattern["name"] == "network-stress-marker" for pattern in patterns):
|
||||
request("POST", "/pattern/", {
|
||||
"name": "network-stress-marker",
|
||||
"value": PAYLOAD_MARKER.hex(),
|
||||
"color": "#ff00ff",
|
||||
"searchType": "SUBBYTES",
|
||||
"searchTarget": "CONTENT",
|
||||
"directionType": "BOTH",
|
||||
"actionType": "FIND",
|
||||
"serviceId": 18080,
|
||||
})
|
||||
if not request("GET", "/pcap/started"):
|
||||
request("POST", "/pcap/start")
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
def probe(
|
||||
requests: int,
|
||||
pause: float,
|
||||
report: str,
|
||||
max_p99_ms: float,
|
||||
stop_file: str | None,
|
||||
min_requests: int,
|
||||
max_queue: int,
|
||||
abort_file: str | None,
|
||||
) -> None:
|
||||
latencies = []
|
||||
failures = []
|
||||
max_capture_queue = 0
|
||||
capture_drops = 0
|
||||
interface_drops = 0
|
||||
application_drops = 0
|
||||
body = {"startingFrom": None, "pageSize": 20, "pattern": None, "favorites": False}
|
||||
for index in range(requests):
|
||||
if stop_file and index >= min_requests and Path(stop_file).exists():
|
||||
break
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
request("POST", "/stream/all", body)
|
||||
latencies.append((time.perf_counter() - started) * 1000)
|
||||
metrics = request("GET", "/pcap/metrics")
|
||||
if not metrics["captureOpen"] or not metrics["statsAvailable"]:
|
||||
raise RuntimeError(f"pcap statistics unavailable: {metrics.get('statsError')}")
|
||||
max_capture_queue = max(max_capture_queue, metrics["maximumQueuedPackets"])
|
||||
capture_drops = max(capture_drops, metrics["droppedPackets"])
|
||||
interface_drops = max(interface_drops, metrics["interfaceDroppedPackets"])
|
||||
application_drops = max(application_drops, metrics["applicationDroppedPackets"])
|
||||
if (
|
||||
metrics["maximumQueuedPackets"] > max_queue
|
||||
or metrics["droppedPackets"] > 0
|
||||
or metrics["interfaceDroppedPackets"] > 0
|
||||
or metrics["applicationDroppedPackets"] > 0
|
||||
):
|
||||
raise RuntimeError(
|
||||
"capture limit exceeded: "
|
||||
f"maxQueueObserved={metrics['maximumQueuedPackets']} maxQueue={max_queue} "
|
||||
f"drops={metrics['droppedPackets']} interfaceDrops={metrics['interfaceDroppedPackets']} "
|
||||
f"applicationDrops={metrics['applicationDroppedPackets']}"
|
||||
)
|
||||
except Exception as exc:
|
||||
failures.append(f"request={index}: {type(exc).__name__}: {exc}")
|
||||
if abort_file:
|
||||
Path(abort_file).touch()
|
||||
break
|
||||
time.sleep(pause)
|
||||
|
||||
latencies.sort()
|
||||
p99 = latencies[max(0, math.ceil(len(latencies) * 0.99) - 1)] if latencies else math.inf
|
||||
result = {
|
||||
"maximumRequests": requests,
|
||||
"requests": len(latencies) + len(failures),
|
||||
"successfulRequests": len(latencies),
|
||||
"failedRequests": len(failures),
|
||||
"latencyP99Millis": round(p99, 3) if latencies else None,
|
||||
"latencyMaxMillis": round(max(latencies, default=0), 3),
|
||||
"maxCaptureQueue": max_capture_queue,
|
||||
"captureDroppedPackets": capture_drops,
|
||||
"interfaceDroppedPackets": interface_drops,
|
||||
"applicationDroppedPackets": application_drops,
|
||||
"sampleErrors": failures[:20],
|
||||
"stoppedByWorkloadCompletion": bool(stop_file and Path(stop_file).exists()),
|
||||
}
|
||||
Path(report).write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(result), flush=True)
|
||||
if (
|
||||
failures
|
||||
or p99 > max_p99_ms
|
||||
or capture_drops > 0
|
||||
or interface_drops > 0
|
||||
or application_drops > 0
|
||||
or max_capture_queue > max_queue
|
||||
or len(latencies) < min_requests
|
||||
or (stop_file and not Path(stop_file).exists())
|
||||
):
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def monitor_capture(
|
||||
report: str,
|
||||
stop_file: str,
|
||||
abort_file: str,
|
||||
max_queue: int,
|
||||
pause: float,
|
||||
min_samples: int,
|
||||
) -> None:
|
||||
samples = 0
|
||||
maximum_queue = 0
|
||||
maximum_drops = 0
|
||||
maximum_interface_drops = 0
|
||||
maximum_application_drops = 0
|
||||
failures = []
|
||||
last_metrics = None
|
||||
started_at = int(time.time() * 1000)
|
||||
|
||||
while True:
|
||||
try:
|
||||
metrics = request("GET", "/pcap/metrics")
|
||||
last_metrics = metrics
|
||||
samples += 1
|
||||
maximum_queue = max(maximum_queue, metrics["maximumQueuedPackets"])
|
||||
maximum_drops = max(maximum_drops, metrics["droppedPackets"])
|
||||
maximum_interface_drops = max(maximum_interface_drops, metrics["interfaceDroppedPackets"])
|
||||
maximum_application_drops = max(
|
||||
maximum_application_drops, metrics["applicationDroppedPackets"]
|
||||
)
|
||||
if not metrics["captureOpen"] or not metrics["statsAvailable"]:
|
||||
raise RuntimeError(f"pcap statistics unavailable: {metrics.get('statsError')}")
|
||||
if (
|
||||
maximum_queue > max_queue
|
||||
or maximum_drops > 0
|
||||
or maximum_interface_drops > 0
|
||||
or maximum_application_drops > 0
|
||||
):
|
||||
raise RuntimeError(
|
||||
"capture limit exceeded: "
|
||||
f"maxQueue={maximum_queue} limit={max_queue} "
|
||||
f"drops={maximum_drops} interfaceDrops={maximum_interface_drops} "
|
||||
f"applicationDrops={maximum_application_drops}"
|
||||
)
|
||||
except Exception as exc:
|
||||
failures.append(f"{type(exc).__name__}: {exc}")
|
||||
Path(abort_file).touch()
|
||||
break
|
||||
|
||||
if samples >= min_samples and Path(stop_file).exists():
|
||||
break
|
||||
time.sleep(pause)
|
||||
|
||||
result = {
|
||||
"startedAtUnixMillis": started_at,
|
||||
"finishedAtUnixMillis": int(time.time() * 1000),
|
||||
"samples": samples,
|
||||
"maxCaptureQueue": maximum_queue,
|
||||
"captureDroppedPackets": maximum_drops,
|
||||
"interfaceDroppedPackets": maximum_interface_drops,
|
||||
"applicationDroppedPackets": maximum_application_drops,
|
||||
"queueLimit": max_queue,
|
||||
"stoppedByWorkloadCompletion": Path(stop_file).exists(),
|
||||
"abortedWorkload": Path(abort_file).exists(),
|
||||
"lastMetrics": last_metrics,
|
||||
"sampleErrors": failures[:20],
|
||||
}
|
||||
Path(report).write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(result), flush=True)
|
||||
if (
|
||||
failures
|
||||
or samples < min_samples
|
||||
or not Path(stop_file).exists()
|
||||
or maximum_queue > max_queue
|
||||
or maximum_drops > 0
|
||||
or maximum_interface_drops > 0
|
||||
or maximum_application_drops > 0
|
||||
):
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def capture_metrics(report: str, require_empty_queue: bool, wait_seconds: int, max_queue: int | None) -> None:
|
||||
deadline = time.monotonic() + wait_seconds
|
||||
while True:
|
||||
metrics = request("GET", "/pcap/metrics")
|
||||
if not require_empty_queue or (metrics["queuedPackets"] == 0 and metrics["activeProcessors"] == 0):
|
||||
break
|
||||
if time.monotonic() >= deadline:
|
||||
break
|
||||
time.sleep(1)
|
||||
Path(report).write_text(json.dumps(metrics, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(metrics), flush=True)
|
||||
if (
|
||||
not metrics["captureOpen"]
|
||||
or not metrics["statsAvailable"]
|
||||
or metrics["receivedPackets"] <= 0
|
||||
or metrics["droppedPackets"] > 0
|
||||
or metrics["interfaceDroppedPackets"] > 0
|
||||
or metrics["applicationDroppedPackets"] > 0
|
||||
or (max_queue is not None and metrics["maximumQueuedPackets"] > max_queue)
|
||||
):
|
||||
raise SystemExit(1)
|
||||
if require_empty_queue and (metrics["queuedPackets"] != 0 or metrics["activeProcessors"] != 0):
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def parse_frames(content: bytes) -> list[bytes]:
|
||||
frames = []
|
||||
offset = 0
|
||||
while offset < len(content):
|
||||
if offset + 4 > len(content):
|
||||
raise ValueError("truncated frame header")
|
||||
size = struct.unpack_from("!I", content, offset)[0]
|
||||
offset += 4
|
||||
if size <= 0 or offset + size > len(content):
|
||||
raise ValueError(f"invalid frame size {size}")
|
||||
frames.append(content[offset:offset + size])
|
||||
offset += size
|
||||
return frames
|
||||
|
||||
|
||||
def get_all_packets(stream_id: int) -> list[dict]:
|
||||
packets = []
|
||||
cursor = None
|
||||
page_size = 1000
|
||||
while True:
|
||||
page = request("POST", f"/packet/{stream_id}", {
|
||||
"startingFrom": cursor,
|
||||
"pageSize": page_size,
|
||||
})
|
||||
packets.extend(page)
|
||||
if len(page) < page_size:
|
||||
return packets
|
||||
new_cursor = page[-1]["id"]
|
||||
if new_cursor == cursor:
|
||||
raise ValueError("packet pagination cursor did not advance")
|
||||
cursor = new_cursor
|
||||
|
||||
|
||||
def verify_payloads(
|
||||
run_id: int,
|
||||
sample_size: int,
|
||||
frames_per_stream: int,
|
||||
payload_bytes: int,
|
||||
report: str,
|
||||
) -> None:
|
||||
streams = request("POST", "/stream/all", {
|
||||
"startingFrom": None,
|
||||
"pageSize": sample_size,
|
||||
"pattern": None,
|
||||
"favorites": False,
|
||||
})
|
||||
failures = []
|
||||
connection_ids = set()
|
||||
verified = 0
|
||||
for stream in streams:
|
||||
try:
|
||||
packets = get_all_packets(stream["id"])
|
||||
incoming = b"".join(base64.b64decode(packet["content"]) for packet in packets if packet["incoming"])
|
||||
outgoing = b"".join(base64.b64decode(packet["content"]) for packet in packets if not packet["incoming"])
|
||||
incoming_frames = parse_frames(incoming)
|
||||
outgoing_frames = parse_frames(outgoing)
|
||||
if len(incoming_frames) != frames_per_stream or len(outgoing_frames) != frames_per_stream:
|
||||
raise ValueError(
|
||||
f"frame count incoming={len(incoming_frames)} outgoing={len(outgoing_frames)}"
|
||||
)
|
||||
|
||||
identities = []
|
||||
for sequence, payload in enumerate(incoming_frames):
|
||||
if len(payload) < 24:
|
||||
raise ValueError("short request payload")
|
||||
frame_run, connection, shard, frame_sequence = struct.unpack("!QQII", payload[:24])
|
||||
if frame_run != run_id or frame_sequence != sequence:
|
||||
raise ValueError(
|
||||
f"unexpected marker run={frame_run} connection={connection} sequence={frame_sequence}"
|
||||
)
|
||||
identities.append((connection, shard))
|
||||
expected = build_payload(run_id, connection, shard, sequence, payload_bytes)
|
||||
if payload != expected:
|
||||
raise ValueError(f"request payload differs at sequence {sequence}")
|
||||
if outgoing_frames[sequence] != expected:
|
||||
raise ValueError(f"response payload differs at sequence {sequence}")
|
||||
if len(set(identities)) != 1:
|
||||
raise ValueError("one captured TCP stream contains multiple generator connections")
|
||||
if identities[0] in connection_ids:
|
||||
raise ValueError("duplicate generator connection across captured streams")
|
||||
|
||||
for packet in packets:
|
||||
if not packet.get("sourceIp") or not packet.get("destinationIp"):
|
||||
raise ValueError("missing packet IP metadata")
|
||||
local_ip = packet["destinationIp"] if packet["incoming"] else packet["sourceIp"]
|
||||
if local_ip != EXPECTED_LOCAL_IP:
|
||||
raise ValueError(f"unexpected local IP {local_ip}")
|
||||
|
||||
connection_ids.add(identities[0])
|
||||
verified += 1
|
||||
except (ValueError, KeyError, struct.error, binascii.Error) as exc:
|
||||
failures.append(f"stream={stream.get('id')}: {type(exc).__name__}: {exc}")
|
||||
|
||||
result = {
|
||||
"runId": run_id,
|
||||
"requestedSample": sample_size,
|
||||
"returnedStreams": len(streams),
|
||||
"verifiedStreams": verified,
|
||||
"failedStreams": len(failures),
|
||||
"sampleErrors": failures[:50],
|
||||
}
|
||||
Path(report).write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(result), flush=True)
|
||||
if len(streams) < sample_size or failures:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
prepare = subparsers.add_parser("prepare")
|
||||
prepare.add_argument("--timeout", type=int, default=300)
|
||||
latency = subparsers.add_parser("probe")
|
||||
latency.add_argument("--requests", type=int, default=300)
|
||||
latency.add_argument("--pause", type=float, default=0.1)
|
||||
latency.add_argument("--report", required=True)
|
||||
latency.add_argument("--max-p99-ms", type=float, default=500)
|
||||
latency.add_argument("--stop-file")
|
||||
latency.add_argument("--min-requests", type=int, default=1)
|
||||
latency.add_argument("--max-queue", type=int, required=True)
|
||||
latency.add_argument("--abort-file")
|
||||
monitor = subparsers.add_parser("monitor")
|
||||
monitor.add_argument("--report", required=True)
|
||||
monitor.add_argument("--stop-file", required=True)
|
||||
monitor.add_argument("--abort-file", required=True)
|
||||
monitor.add_argument("--max-queue", type=int, required=True)
|
||||
monitor.add_argument("--pause", type=float, default=0.25)
|
||||
monitor.add_argument("--min-samples", type=int, default=2)
|
||||
metrics = subparsers.add_parser("metrics")
|
||||
metrics.add_argument("--report", required=True)
|
||||
metrics.add_argument("--require-empty-queue", action="store_true")
|
||||
metrics.add_argument("--wait-seconds", type=int, default=0)
|
||||
metrics.add_argument("--max-queue", type=int)
|
||||
verify = subparsers.add_parser("verify")
|
||||
verify.add_argument("--run-id", type=int, required=True)
|
||||
verify.add_argument("--sample-size", type=int, default=200)
|
||||
verify.add_argument("--frames", type=int, required=True)
|
||||
verify.add_argument("--payload-bytes", type=int, required=True)
|
||||
verify.add_argument("--report", required=True)
|
||||
args = parser.parse_args()
|
||||
if args.command == "prepare":
|
||||
wait_and_prepare(args.timeout)
|
||||
elif args.command == "probe":
|
||||
probe(
|
||||
args.requests,
|
||||
args.pause,
|
||||
args.report,
|
||||
args.max_p99_ms,
|
||||
args.stop_file,
|
||||
args.min_requests,
|
||||
args.max_queue,
|
||||
args.abort_file,
|
||||
)
|
||||
elif args.command == "monitor":
|
||||
monitor_capture(
|
||||
args.report,
|
||||
args.stop_file,
|
||||
args.abort_file,
|
||||
args.max_queue,
|
||||
args.pause,
|
||||
args.min_samples,
|
||||
)
|
||||
elif args.command == "metrics":
|
||||
capture_metrics(args.report, args.require_empty_queue, args.wait_seconds, args.max_queue)
|
||||
else:
|
||||
verify_payloads(args.run_id, args.sample_size, args.frames, args.payload_bytes, args.report)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
138
stress/network/docker-compose.yml
Normal file
@ -0,0 +1,138 @@
|
||||
services:
|
||||
db:
|
||||
image: postgres:15.2
|
||||
shm_size: 1gb
|
||||
command:
|
||||
- postgres
|
||||
- -c
|
||||
- max_connections=200
|
||||
- -c
|
||||
- autovacuum_max_workers=4
|
||||
- -c
|
||||
- autovacuum_naptime=2s
|
||||
- -c
|
||||
- autovacuum_vacuum_threshold=1000
|
||||
- -c
|
||||
- autovacuum_vacuum_scale_factor=0.02
|
||||
- -c
|
||||
- autovacuum_vacuum_cost_limit=1000
|
||||
environment:
|
||||
POSTGRES_DB: packmate
|
||||
POSTGRES_USER: packmate
|
||||
POSTGRES_PASSWORD: network-stress
|
||||
volumes:
|
||||
- db_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U packmate -d packmate"]
|
||||
interval: 1s
|
||||
timeout: 3s
|
||||
retries: 60
|
||||
networks:
|
||||
stress:
|
||||
ipv4_address: ${PACKMATE_NETWORK_STRESS_DB_IP:-172.31.250.20}
|
||||
|
||||
target:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.tools
|
||||
command: ["python", "/stress/tcp_target.py"]
|
||||
environment:
|
||||
STRESS_TARGET_HOST: 0.0.0.0
|
||||
STRESS_TARGET_PORT: 18080
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import socket; s=socket.create_connection(('127.0.0.1',18080),2); s.close()"]
|
||||
interval: 1s
|
||||
timeout: 3s
|
||||
retries: 60
|
||||
networks:
|
||||
stress:
|
||||
ipv4_address: ${PACKMATE_NETWORK_STRESS_TARGET_IP:-172.31.250.10}
|
||||
|
||||
packmate:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: docker/Dockerfile_app
|
||||
network_mode: service:target
|
||||
stop_grace_period: 45s
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
- NET_RAW
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
target:
|
||||
condition: service_healthy
|
||||
command:
|
||||
- java
|
||||
- -Xms512m
|
||||
- -Xmx${PACKMATE_NETWORK_STRESS_JAVA_MAX_HEAP:-4g}
|
||||
- -Djava.net.preferIPv4Stack=true
|
||||
- -Djava.net.preferIPv4Addresses=true
|
||||
- -jar
|
||||
- /app/app.jar
|
||||
- --spring.datasource.url=jdbc:postgresql://db:5432/packmate
|
||||
- --spring.datasource.password=network-stress
|
||||
- --packmate.capture-mode=LIVE
|
||||
- --packmate.interface-name=eth0
|
||||
- --packmate.local-ip=${PACKMATE_NETWORK_STRESS_TARGET_IP:-172.31.250.10}
|
||||
- --packmate.web.account-login=stress
|
||||
- --packmate.web.account-password=stress
|
||||
- --packmate.timeout.tcp-stream-timeout=${PACKMATE_NETWORK_STRESS_TCP_TIMEOUT_SECONDS:-40}
|
||||
- --packmate.timeout.udp-stream-timeout=${PACKMATE_NETWORK_STRESS_UDP_TIMEOUT_SECONDS:-20}
|
||||
- --packmate.timeout.check-interval=1
|
||||
- --packmate.capture.workers=${PACKMATE_NETWORK_STRESS_CAPTURE_WORKERS:-8}
|
||||
- --packmate.capture.queue-capacity=${PACKMATE_NETWORK_STRESS_CAPTURE_QUEUE_CAPACITY:-250000}
|
||||
- --packmate.capture.buffer-size-bytes=${PACKMATE_NETWORK_STRESS_CAPTURE_BUFFER_SIZE_BYTES:-67108864}
|
||||
- --packmate.capture.shutdown-timeout-seconds=30
|
||||
- --packmate.cleanup.enabled=${PACKMATE_NETWORK_STRESS_CLEANUP_ENABLED:-false}
|
||||
- --packmate.cleanup.threshold=${PACKMATE_NETWORK_STRESS_CLEANUP_THRESHOLD_MINUTES:-1440}
|
||||
- --packmate.cleanup.interval-seconds=${PACKMATE_NETWORK_STRESS_CLEANUP_INTERVAL_SECONDS:-10}
|
||||
- --packmate.cleanup.max-run-seconds=5
|
||||
- --packmate.cleanup.commit-grace-seconds=120
|
||||
- --packmate.cleanup.batch-size=500
|
||||
- --packmate.cleanup.max-batches-per-run=20
|
||||
- --packmate.cleanup.max-packets-per-batch=20000
|
||||
- --packmate.cleanup.max-matches-per-batch=20000
|
||||
- --packmate.cleanup.max-bytes-per-batch=67108864
|
||||
- --packmate.cleanup.statement-timeout-seconds=5
|
||||
- --packmate.cleanup.lock-timeout-seconds=1
|
||||
- --server.port=65000
|
||||
- --server.address=0.0.0.0
|
||||
|
||||
generator:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.tools
|
||||
profiles: ["tools"]
|
||||
working_dir: /stress
|
||||
environment:
|
||||
STRESS_TARGET_HOST: ${PACKMATE_NETWORK_STRESS_TARGET_IP:-172.31.250.10}
|
||||
STRESS_TARGET_PORT: 18080
|
||||
STRESS_API_URL: http://${PACKMATE_NETWORK_STRESS_TARGET_IP:-172.31.250.10}:65000/api
|
||||
STRESS_API_USER: stress
|
||||
STRESS_API_PASSWORD: stress
|
||||
volumes:
|
||||
- reports:/reports
|
||||
networks:
|
||||
- stress
|
||||
|
||||
reports:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.tools
|
||||
profiles: ["tools"]
|
||||
command: ["sleep", "infinity"]
|
||||
volumes:
|
||||
- reports:/reports
|
||||
networks:
|
||||
- stress
|
||||
|
||||
networks:
|
||||
stress:
|
||||
ipam:
|
||||
config:
|
||||
- subnet: ${PACKMATE_NETWORK_STRESS_SUBNET:-172.31.250.0/24}
|
||||
|
||||
volumes:
|
||||
reports:
|
||||
db_data:
|
||||
161
stress/network/network_load.py
Normal file
@ -0,0 +1,161 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import socket
|
||||
import struct
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
PAYLOAD_MARKER = b"PMSTRESS"
|
||||
|
||||
|
||||
def build_payload(run_id: int, stream_number: int, shard: int, sequence: int, payload_bytes: int) -> bytes:
|
||||
prefix = struct.pack("!QQII", run_id, stream_number, shard, sequence)
|
||||
padding = bytearray(
|
||||
((stream_number + sequence + i) % 251 for i in range(payload_bytes - len(prefix)))
|
||||
)
|
||||
if sequence == 0:
|
||||
padding[:len(PAYLOAD_MARKER)] = PAYLOAD_MARKER
|
||||
return prefix + padding
|
||||
|
||||
|
||||
def percentile(values: list[float], fraction: float) -> float:
|
||||
if not values:
|
||||
return math.inf
|
||||
values.sort()
|
||||
return values[max(0, math.ceil(len(values) * fraction) - 1)]
|
||||
|
||||
|
||||
async def send_stream(
|
||||
stream_number: int,
|
||||
host: str,
|
||||
port: int,
|
||||
frames: int,
|
||||
payload_bytes: int,
|
||||
run_id: int,
|
||||
shard: int,
|
||||
) -> float:
|
||||
started = time.perf_counter()
|
||||
reader, writer = await asyncio.wait_for(asyncio.open_connection(host, port), timeout=10)
|
||||
sock = writer.get_extra_info("socket")
|
||||
if sock is not None:
|
||||
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
||||
|
||||
try:
|
||||
for sequence in range(frames):
|
||||
payload = build_payload(run_id, stream_number, shard, sequence, payload_bytes)
|
||||
writer.write(struct.pack("!I", len(payload)) + payload)
|
||||
await writer.drain()
|
||||
response_size = struct.unpack("!I", await asyncio.wait_for(reader.readexactly(4), 10))[0]
|
||||
response = await asyncio.wait_for(reader.readexactly(response_size), 10)
|
||||
if response_size != len(payload) or response != payload:
|
||||
raise ValueError(
|
||||
f"invalid echo for sequence={sequence}: expected={len(payload)} actual={response_size}"
|
||||
)
|
||||
finally:
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
return (time.perf_counter() - started) * 1000
|
||||
|
||||
|
||||
async def run(args: argparse.Namespace) -> dict:
|
||||
next_stream = 0
|
||||
counter_lock = asyncio.Lock()
|
||||
latencies: list[float] = []
|
||||
errors: list[str] = []
|
||||
abort_file = Path(args.abort_file) if args.abort_file else None
|
||||
|
||||
async def worker() -> None:
|
||||
nonlocal next_stream
|
||||
while True:
|
||||
async with counter_lock:
|
||||
if abort_file and abort_file.exists():
|
||||
return
|
||||
if next_stream >= args.streams:
|
||||
return
|
||||
local_number = next_stream
|
||||
next_stream += 1
|
||||
if args.minimum_duration_seconds > 0:
|
||||
scheduled_at = started + (local_number * args.minimum_duration_seconds / args.streams)
|
||||
delay = scheduled_at - time.perf_counter()
|
||||
if delay > 0:
|
||||
await asyncio.sleep(delay)
|
||||
stream_number = args.stream_offset + local_number
|
||||
try:
|
||||
latency = await send_stream(
|
||||
stream_number,
|
||||
args.host,
|
||||
args.port,
|
||||
args.frames,
|
||||
args.payload_bytes,
|
||||
args.run_id,
|
||||
args.shard,
|
||||
)
|
||||
latencies.append(latency)
|
||||
except Exception as exc: # individual failures are reported and bounded by the success threshold
|
||||
if len(errors) < 100:
|
||||
errors.append(f"stream={stream_number}: {type(exc).__name__}: {exc}")
|
||||
|
||||
started_at_unix_seconds = time.time()
|
||||
started = time.perf_counter()
|
||||
await asyncio.gather(*(worker() for _ in range(args.concurrency)))
|
||||
elapsed = time.perf_counter() - started
|
||||
finished_at_unix_seconds = time.time()
|
||||
successes = len(latencies)
|
||||
result = {
|
||||
"runId": args.run_id,
|
||||
"shard": args.shard,
|
||||
"requestedStreams": args.streams,
|
||||
"successfulStreams": successes,
|
||||
"failedStreams": args.streams - successes,
|
||||
"framesPerStream": args.frames,
|
||||
"payloadBytes": args.payload_bytes,
|
||||
"minimumDurationSeconds": args.minimum_duration_seconds,
|
||||
"startedAtUnixSeconds": round(started_at_unix_seconds, 6),
|
||||
"finishedAtUnixSeconds": round(finished_at_unix_seconds, 6),
|
||||
"elapsedSeconds": round(elapsed, 3),
|
||||
"streamsPerSecond": round(successes / elapsed, 3) if elapsed else 0,
|
||||
"latencyP50Millis": round(percentile(latencies, 0.50), 3) if latencies else None,
|
||||
"latencyP95Millis": round(percentile(latencies, 0.95), 3) if latencies else None,
|
||||
"latencyP99Millis": round(percentile(latencies, 0.99), 3) if latencies else None,
|
||||
"latencyMaxMillis": round(max(latencies, default=0), 3),
|
||||
"abortedByCaptureMonitor": bool(abort_file and abort_file.exists()),
|
||||
"sampleErrors": errors,
|
||||
}
|
||||
Path(args.report).write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(result), flush=True)
|
||||
return result
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Generate real framed TCP traffic for Packmate")
|
||||
parser.add_argument("--host", default=os.getenv("STRESS_TARGET_HOST", "172.31.250.10"))
|
||||
parser.add_argument("--port", type=int, default=int(os.getenv("STRESS_TARGET_PORT", "18080")))
|
||||
parser.add_argument("--streams", type=int, required=True)
|
||||
parser.add_argument("--frames", type=int, default=8)
|
||||
parser.add_argument("--payload-bytes", type=int, default=1024)
|
||||
parser.add_argument("--concurrency", type=int, default=256)
|
||||
parser.add_argument("--run-id", type=int, required=True)
|
||||
parser.add_argument("--shard", type=int, required=True)
|
||||
parser.add_argument("--stream-offset", type=int, required=True)
|
||||
parser.add_argument("--report", required=True)
|
||||
parser.add_argument("--abort-file")
|
||||
parser.add_argument("--minimum-duration-seconds", type=float, default=0)
|
||||
args = parser.parse_args()
|
||||
if (
|
||||
args.streams <= 0
|
||||
or args.frames <= 0
|
||||
or args.payload_bytes < 32
|
||||
or args.concurrency <= 0
|
||||
or args.minimum_duration_seconds < 0
|
||||
):
|
||||
parser.error("streams, frames and concurrency must be positive; payload must be at least 32 bytes")
|
||||
return args
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
arguments = parse_args()
|
||||
outcome = asyncio.run(run(arguments))
|
||||
raise SystemExit(0 if outcome["successfulStreams"] == arguments.streams else 1)
|
||||
650
stress/network/run.sh
Normal file
@ -0,0 +1,650 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
REPO_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/../.." && pwd)
|
||||
COMPOSE_FILE="$SCRIPT_DIR/docker-compose.yml"
|
||||
PROJECT_RAW=${PACKMATE_NETWORK_STRESS_PROJECT:-"packmate-network-stress-${CI_JOB_ID:-local}"}
|
||||
PROJECT=$(printf '%s' "$PROJECT_RAW" | tr -cd 'a-zA-Z0-9_-' | tr 'A-Z' 'a-z')
|
||||
RUN_LABEL_RAW=${PACKMATE_NETWORK_STRESS_RUN_ID:-"$(date -u +%Y%m%dT%H%M%SZ)-$$"}
|
||||
RUN_LABEL=$(printf '%s' "$RUN_LABEL_RAW" | tr -cd 'a-zA-Z0-9_.-')
|
||||
if [ -z "$PROJECT" ] || [ -z "$RUN_LABEL" ]; then
|
||||
echo "stress project and run ID must contain letters, digits, dot, underscore or dash" >&2
|
||||
exit 2
|
||||
fi
|
||||
REPORT_DIR="$REPO_DIR/build/reports/network-stress/$RUN_LABEL"
|
||||
REPORT_DIR_DOCKER="$REPORT_DIR"
|
||||
if [ -n "${MSYSTEM:-}" ]; then
|
||||
COMPOSE_FILE=$(cygpath -w "$COMPOSE_FILE")
|
||||
REPORT_DIR_DOCKER=$(cygpath -w "$REPORT_DIR")
|
||||
fi
|
||||
|
||||
BASELINE_STREAMS=${PACKMATE_NETWORK_STRESS_STREAMS:-250000}
|
||||
LIVE_STREAMS=${PACKMATE_NETWORK_STRESS_LIVE_STREAMS:-50000}
|
||||
SHARDS=${PACKMATE_NETWORK_STRESS_SHARDS:-16}
|
||||
CONCURRENCY=${PACKMATE_NETWORK_STRESS_CONCURRENCY:-128}
|
||||
FRAMES=${PACKMATE_NETWORK_STRESS_FRAMES_PER_STREAM:-8}
|
||||
PAYLOAD_BYTES=${PACKMATE_NETWORK_STRESS_PAYLOAD_BYTES:-1024}
|
||||
DRAIN_TIMEOUT=${PACKMATE_NETWORK_STRESS_DRAIN_TIMEOUT_SECONDS:-1800}
|
||||
CLEANUP_TIMEOUT=${PACKMATE_NETWORK_STRESS_CLEANUP_TIMEOUT_SECONDS:-1800}
|
||||
MAX_API_P99_MS=${PACKMATE_NETWORK_STRESS_MAX_API_P99_MS:-500}
|
||||
MAX_CAPTURE_QUEUE=${PACKMATE_NETWORK_STRESS_MAX_CAPTURE_QUEUE:-500000}
|
||||
CAPTURE_WORKERS=${PACKMATE_NETWORK_STRESS_CAPTURE_WORKERS:-8}
|
||||
CAPTURE_QUEUE_CAPACITY=${PACKMATE_NETWORK_STRESS_CAPTURE_QUEUE_CAPACITY:-250000}
|
||||
CAPTURE_BUFFER_SIZE_BYTES=${PACKMATE_NETWORK_STRESS_CAPTURE_BUFFER_SIZE_BYTES:-67108864}
|
||||
TCP_TIMEOUT_SECONDS=${PACKMATE_NETWORK_STRESS_TCP_TIMEOUT_SECONDS:-40}
|
||||
UDP_TIMEOUT_SECONDS=${PACKMATE_NETWORK_STRESS_UDP_TIMEOUT_SECONDS:-20}
|
||||
MIN_STREAMS_PER_SECOND=${PACKMATE_NETWORK_STRESS_MIN_STREAMS_PER_SECOND:-500}
|
||||
MIN_CLEANUP_STREAMS_PER_SECOND=${PACKMATE_NETWORK_STRESS_MIN_CLEANUP_STREAMS_PER_SECOND:-$MIN_STREAMS_PER_SECOND}
|
||||
MAX_INGEST_DRAIN_SECONDS=${PACKMATE_NETWORK_STRESS_MAX_INGEST_DRAIN_SECONDS:-120}
|
||||
MIN_LIVE_DURATION_SECONDS=${PACKMATE_NETWORK_STRESS_MIN_LIVE_DURATION_SECONDS:-20}
|
||||
FAVORITE_STREAMS=${PACKMATE_NETWORK_STRESS_FAVORITE_STREAMS:-100}
|
||||
STORAGE_MULTIPLIER=${PACKMATE_NETWORK_STRESS_STORAGE_MULTIPLIER:-3}
|
||||
TARGET_IP=${PACKMATE_NETWORK_STRESS_TARGET_IP:-172.31.250.10}
|
||||
PACKMATE_NETWORK_STRESS_CLEANUP_INTERVAL_SECONDS=${PACKMATE_NETWORK_STRESS_CLEANUP_INTERVAL_SECONDS:-10}
|
||||
PACKMATE_NETWORK_STRESS_CLEANUP_THRESHOLD_MINUTES=${PACKMATE_NETWORK_STRESS_CLEANUP_THRESHOLD_MINUTES:-1440}
|
||||
PACKMATE_NETWORK_STRESS_CAPTURE_WORKERS=$CAPTURE_WORKERS
|
||||
PACKMATE_NETWORK_STRESS_CAPTURE_QUEUE_CAPACITY=$CAPTURE_QUEUE_CAPACITY
|
||||
PACKMATE_NETWORK_STRESS_CAPTURE_BUFFER_SIZE_BYTES=$CAPTURE_BUFFER_SIZE_BYTES
|
||||
PACKMATE_NETWORK_STRESS_TCP_TIMEOUT_SECONDS=$TCP_TIMEOUT_SECONDS
|
||||
PACKMATE_NETWORK_STRESS_UDP_TIMEOUT_SECONDS=$UDP_TIMEOUT_SECONDS
|
||||
export PACKMATE_NETWORK_STRESS_CLEANUP_INTERVAL_SECONDS PACKMATE_NETWORK_STRESS_CLEANUP_THRESHOLD_MINUTES
|
||||
export PACKMATE_NETWORK_STRESS_CAPTURE_WORKERS PACKMATE_NETWORK_STRESS_CAPTURE_QUEUE_CAPACITY
|
||||
export PACKMATE_NETWORK_STRESS_CAPTURE_BUFFER_SIZE_BYTES
|
||||
export PACKMATE_NETWORK_STRESS_TCP_TIMEOUT_SECONDS PACKMATE_NETWORK_STRESS_UDP_TIMEOUT_SECONDS
|
||||
PACKMATE_NETWORK_STRESS_CLEANUP_ENABLED=false
|
||||
export PACKMATE_NETWORK_STRESS_CLEANUP_ENABLED
|
||||
|
||||
require_positive() {
|
||||
case "$1" in ''|*[!0-9]*|0) echo "$2 must be a positive integer" >&2; exit 2 ;; esac
|
||||
}
|
||||
|
||||
for setting in \
|
||||
"$BASELINE_STREAMS:PACKMATE_NETWORK_STRESS_STREAMS" \
|
||||
"$LIVE_STREAMS:PACKMATE_NETWORK_STRESS_LIVE_STREAMS" \
|
||||
"$SHARDS:PACKMATE_NETWORK_STRESS_SHARDS" \
|
||||
"$CONCURRENCY:PACKMATE_NETWORK_STRESS_CONCURRENCY" \
|
||||
"$FRAMES:PACKMATE_NETWORK_STRESS_FRAMES_PER_STREAM" \
|
||||
"$PAYLOAD_BYTES:PACKMATE_NETWORK_STRESS_PAYLOAD_BYTES" \
|
||||
"$MAX_CAPTURE_QUEUE:PACKMATE_NETWORK_STRESS_MAX_CAPTURE_QUEUE" \
|
||||
"$CAPTURE_WORKERS:PACKMATE_NETWORK_STRESS_CAPTURE_WORKERS" \
|
||||
"$CAPTURE_QUEUE_CAPACITY:PACKMATE_NETWORK_STRESS_CAPTURE_QUEUE_CAPACITY" \
|
||||
"$CAPTURE_BUFFER_SIZE_BYTES:PACKMATE_NETWORK_STRESS_CAPTURE_BUFFER_SIZE_BYTES" \
|
||||
"$TCP_TIMEOUT_SECONDS:PACKMATE_NETWORK_STRESS_TCP_TIMEOUT_SECONDS" \
|
||||
"$UDP_TIMEOUT_SECONDS:PACKMATE_NETWORK_STRESS_UDP_TIMEOUT_SECONDS" \
|
||||
"$MIN_STREAMS_PER_SECOND:PACKMATE_NETWORK_STRESS_MIN_STREAMS_PER_SECOND" \
|
||||
"$MIN_CLEANUP_STREAMS_PER_SECOND:PACKMATE_NETWORK_STRESS_MIN_CLEANUP_STREAMS_PER_SECOND" \
|
||||
"$MAX_INGEST_DRAIN_SECONDS:PACKMATE_NETWORK_STRESS_MAX_INGEST_DRAIN_SECONDS" \
|
||||
"$MIN_LIVE_DURATION_SECONDS:PACKMATE_NETWORK_STRESS_MIN_LIVE_DURATION_SECONDS" \
|
||||
"$FAVORITE_STREAMS:PACKMATE_NETWORK_STRESS_FAVORITE_STREAMS" \
|
||||
"$STORAGE_MULTIPLIER:PACKMATE_NETWORK_STRESS_STORAGE_MULTIPLIER" \
|
||||
"$PACKMATE_NETWORK_STRESS_CLEANUP_INTERVAL_SECONDS:PACKMATE_NETWORK_STRESS_CLEANUP_INTERVAL_SECONDS" \
|
||||
"$PACKMATE_NETWORK_STRESS_CLEANUP_THRESHOLD_MINUTES:PACKMATE_NETWORK_STRESS_CLEANUP_THRESHOLD_MINUTES"
|
||||
do
|
||||
require_positive "${setting%%:*}" "${setting#*:}"
|
||||
done
|
||||
if [ "$PAYLOAD_BYTES" -lt 32 ]; then
|
||||
echo "PACKMATE_NETWORK_STRESS_PAYLOAD_BYTES must be at least 32" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [ "$FAVORITE_STREAMS" -gt "$BASELINE_STREAMS" ]; then
|
||||
echo "favorite stream count cannot exceed baseline streams" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
compose() {
|
||||
MSYS_NO_PATHCONV=1 docker compose --profile tools -p "$PROJECT" -f "$COMPOSE_FILE" "$@"
|
||||
}
|
||||
|
||||
db_scalar() {
|
||||
output=$(compose exec -T db psql -v ON_ERROR_STOP=1 -U packmate -d packmate -Atqc "$1") || return $?
|
||||
printf '%s' "$output" | tr -d '\r'
|
||||
}
|
||||
|
||||
db_integer() {
|
||||
value=$(db_scalar "$1") || return $?
|
||||
case "$value" in
|
||||
''|*[!0-9]*) echo "database query did not return an unsigned integer: $value" >&2; return 1 ;;
|
||||
esac
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
save_logs() {
|
||||
compose logs --no-color > "$REPORT_DIR/compose.log" 2>&1 || true
|
||||
}
|
||||
|
||||
copy_reports() {
|
||||
if ! reports_container=$(compose ps -aq reports 2>/dev/null); then
|
||||
echo "could not locate the stress reports container; preserving Docker volumes for recovery" >&2
|
||||
return 1
|
||||
fi
|
||||
if [ -z "$reports_container" ]; then
|
||||
echo "stress reports container does not exist; preserving Docker volumes for recovery" >&2
|
||||
return 1
|
||||
fi
|
||||
if ! MSYS_NO_PATHCONV=1 docker cp "$reports_container:/reports/." "$REPORT_DIR_DOCKER"; then
|
||||
echo "could not copy stress reports; preserving Docker volumes for recovery" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
status=$1
|
||||
trap - 0 2 15
|
||||
reports_copied=true
|
||||
if ! copy_reports; then
|
||||
reports_copied=false
|
||||
status=1
|
||||
fi
|
||||
save_logs
|
||||
if [ "$reports_copied" = true ]; then
|
||||
compose down --volumes --remove-orphans >/dev/null 2>&1 || true
|
||||
else
|
||||
compose down --remove-orphans >/dev/null 2>&1 || true
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
trap 'cleanup $?' 0
|
||||
trap 'cleanup 130' 2
|
||||
trap 'cleanup 143' 15
|
||||
|
||||
wait_for_streams() {
|
||||
condition=$1
|
||||
minimum=$2
|
||||
timeout=$3
|
||||
label=$4
|
||||
watch_pid=${5:-}
|
||||
deadline=$(( $(date +%s) + timeout ))
|
||||
while :; do
|
||||
if [ -n "$watch_pid" ] && ! kill -0 "$watch_pid" 2>/dev/null; then
|
||||
echo "$label stopped because its health probe exited early" >&2
|
||||
return 1
|
||||
fi
|
||||
current=$(db_integer "SELECT count(*) FROM stream WHERE $condition")
|
||||
if [ "$current" -ge "$minimum" ]; then
|
||||
printf '%s reached %s streams (required %s)\n' "$label" "$current" "$minimum"
|
||||
return 0
|
||||
fi
|
||||
if [ "$(date +%s)" -ge "$deadline" ]; then
|
||||
echo "$label reached only $current of required $minimum streams" >&2
|
||||
return 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
wait_for_zero() {
|
||||
condition=$1
|
||||
timeout=$2
|
||||
label=$3
|
||||
watch_pid=${4:-}
|
||||
deadline=$(( $(date +%s) + timeout ))
|
||||
while :; do
|
||||
if [ -n "$watch_pid" ] && ! kill -0 "$watch_pid" 2>/dev/null; then
|
||||
echo "$label stopped because its health probe exited early" >&2
|
||||
return 1
|
||||
fi
|
||||
current=$(db_integer "SELECT count(*) FROM stream WHERE $condition")
|
||||
if [ "$current" -eq 0 ]; then
|
||||
printf '%s drained\n' "$label"
|
||||
return 0
|
||||
fi
|
||||
if [ "$(date +%s)" -ge "$deadline" ]; then
|
||||
echo "$label still has $current streams" >&2
|
||||
return 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
check_rate() {
|
||||
phase=$1
|
||||
streams=$2
|
||||
elapsed=$3
|
||||
if [ "$elapsed" -lt 1 ]; then elapsed=1; fi
|
||||
if [ "$streams" -lt $((MIN_STREAMS_PER_SECOND * elapsed)) ]; then
|
||||
echo "$phase achieved less than $MIN_STREAMS_PER_SECOND completed TCP streams/s ($streams in ${elapsed}s)" >&2
|
||||
return 1
|
||||
fi
|
||||
printf '%s completed %s streams in %ss (at least %s streams/s)\n' \
|
||||
"$phase" "$streams" "$elapsed" "$MIN_STREAMS_PER_SECOND"
|
||||
}
|
||||
|
||||
load_phase_elapsed() {
|
||||
phase=$1
|
||||
compose exec -T reports python -c '
|
||||
import glob, json, math, sys
|
||||
paths = glob.glob(f"/reports/{sys.argv[1]}-*.json")
|
||||
if not paths:
|
||||
raise SystemExit(f"no generator reports found for phase {sys.argv[1]}")
|
||||
reports = [json.load(open(path, encoding="utf-8")) for path in paths]
|
||||
reports = [report for report in reports if "startedAtUnixSeconds" in report]
|
||||
if not reports:
|
||||
raise SystemExit(f"no timed generator reports found for phase {sys.argv[1]}")
|
||||
elapsed = max(report["finishedAtUnixSeconds"] for report in reports) \
|
||||
- min(report["startedAtUnixSeconds"] for report in reports)
|
||||
print(max(1, math.ceil(elapsed)))
|
||||
' "$phase" | tr -d '\r'
|
||||
}
|
||||
|
||||
emit_ledger() {
|
||||
stream_condition=$1
|
||||
compose exec -T db psql -v ON_ERROR_STOP=1 -U packmate -d packmate -At \
|
||||
-F "$(printf '\t')" -c "
|
||||
WITH payloads AS (
|
||||
SELECT s.id AS stream_id,
|
||||
string_agg(p.content, ''::bytea ORDER BY p.id) FILTER (WHERE p.incoming) AS incoming_content,
|
||||
string_agg(p.content, ''::bytea ORDER BY p.id) FILTER (WHERE NOT p.incoming) AS outgoing_content,
|
||||
bool_and(COALESCE(
|
||||
CASE WHEN p.incoming
|
||||
THEN p.destination_ip = '$TARGET_IP' AND p.source_ip <> '$TARGET_IP'
|
||||
ELSE p.source_ip = '$TARGET_IP' AND p.destination_ip <> '$TARGET_IP'
|
||||
END,
|
||||
false
|
||||
)) AS ips_valid,
|
||||
count(p.id) = s.packets_count
|
||||
AND sum(octet_length(p.content)) = s.size_bytes
|
||||
AND s.service_id = 18080
|
||||
AND s.protocol = 0
|
||||
AND (s.end_timestamp = 0 OR s.start_timestamp <= s.end_timestamp)
|
||||
AS metadata_valid
|
||||
FROM stream s
|
||||
JOIN packet p ON p.stream_id = s.id
|
||||
WHERE $stream_condition
|
||||
GROUP BY s.id, s.packets_count, s.size_bytes, s.service_id, s.protocol,
|
||||
s.start_timestamp, s.end_timestamp
|
||||
)
|
||||
SELECT stream_id,
|
||||
encode(substring(incoming_content FROM 5 FOR 24), 'hex'),
|
||||
octet_length(incoming_content),
|
||||
md5(incoming_content),
|
||||
octet_length(outgoing_content),
|
||||
md5(outgoing_content),
|
||||
ips_valid,
|
||||
metadata_valid
|
||||
FROM payloads
|
||||
ORDER BY stream_id"
|
||||
}
|
||||
|
||||
count_invalid_matches() {
|
||||
stream_condition=$1
|
||||
db_integer "
|
||||
SELECT count(*)
|
||||
FROM (
|
||||
SELECT s.id
|
||||
FROM stream s
|
||||
JOIN packet p ON p.stream_id = s.id
|
||||
LEFT JOIN found_pattern fp ON fp.packet_id = p.id
|
||||
WHERE $stream_condition
|
||||
GROUP BY s.id
|
||||
HAVING count(fp.id) <> 2
|
||||
OR count(fp.id) FILTER (WHERE p.incoming) <> 1
|
||||
OR count(fp.id) FILTER (WHERE NOT p.incoming) <> 1
|
||||
OR bool_and(
|
||||
substring(
|
||||
p.content
|
||||
FROM fp.start_position + 1
|
||||
FOR fp.end_position - fp.start_position + 1
|
||||
) = convert_to('PMSTRESS', 'UTF8')
|
||||
AND COALESCE(fp.search_target, 'CONTENT') = 'CONTENT'
|
||||
AND fp.pattern_id = (SELECT id FROM pattern WHERE name = 'network-stress-marker')
|
||||
) FILTER (WHERE fp.id IS NOT NULL) IS NOT TRUE
|
||||
) invalid"
|
||||
}
|
||||
|
||||
verify_complete_ledger() {
|
||||
condition=$1
|
||||
run_id=$2
|
||||
total=$3
|
||||
expected=$4
|
||||
report=$5
|
||||
subset=${6:-false}
|
||||
subset_arg=""
|
||||
ledger_file="$REPORT_DIR/$report.tsv"
|
||||
if [ "$subset" = true ]; then subset_arg="--allow-subset"; fi
|
||||
emit_ledger "$condition" > "$ledger_file"
|
||||
# shellcheck disable=SC2086
|
||||
compose run --rm -T --no-deps generator \
|
||||
python /stress/verify_ledger.py \
|
||||
--run-id "$run_id" \
|
||||
--total-streams "$total" \
|
||||
--expected-streams "$expected" \
|
||||
--shards "$SHARDS" \
|
||||
--frames "$FRAMES" \
|
||||
--payload-bytes "$PAYLOAD_BYTES" \
|
||||
--report "/reports/$report" \
|
||||
$subset_arg < "$ledger_file"
|
||||
rm -f "$ledger_file"
|
||||
}
|
||||
|
||||
run_load_phase() {
|
||||
phase=$1
|
||||
total=$2
|
||||
run_id=$3
|
||||
abort_file=$4
|
||||
minimum_duration=$5
|
||||
base=$((total / SHARDS))
|
||||
remainder=$((total % SHARDS))
|
||||
pids=""
|
||||
shard=0
|
||||
offset=0
|
||||
while [ "$shard" -lt "$SHARDS" ]; do
|
||||
count=$base
|
||||
if [ "$shard" -lt "$remainder" ]; then count=$((count + 1)); fi
|
||||
if [ "$count" -gt 0 ]; then
|
||||
compose run --rm --no-deps generator python /stress/network_load.py \
|
||||
--streams "$count" \
|
||||
--frames "$FRAMES" \
|
||||
--payload-bytes "$PAYLOAD_BYTES" \
|
||||
--concurrency "$CONCURRENCY" \
|
||||
--run-id "$run_id" \
|
||||
--shard "$shard" \
|
||||
--stream-offset "$offset" \
|
||||
--abort-file "$abort_file" \
|
||||
--minimum-duration-seconds "$minimum_duration" \
|
||||
--report "/reports/$phase-$shard.json" &
|
||||
pids="$pids $!"
|
||||
fi
|
||||
offset=$((offset + count))
|
||||
shard=$((shard + 1))
|
||||
done
|
||||
failed=0
|
||||
for pid in $pids; do
|
||||
if ! wait "$pid"; then failed=1; fi
|
||||
done
|
||||
return "$failed"
|
||||
}
|
||||
|
||||
echo "Starting isolated network stress project $PROJECT; reports: $REPORT_DIR"
|
||||
compose up -d --build db target packmate reports
|
||||
compose build generator
|
||||
compose exec -T reports sh -c 'rm -f /reports/*'
|
||||
compose run --rm --no-deps generator python /stress/api_control.py prepare --timeout 300
|
||||
|
||||
db_df=$(compose exec -T db df -PkT /var/lib/postgresql/data) || exit $?
|
||||
db_available_kb=$(printf '%s\n' "$db_df" | awk 'NR == 2 {print $5}' | tr -d '\r')
|
||||
db_storage_type=$(printf '%s\n' "$db_df" | awk 'NR == 2 {print $2}' | tr -d '\r')
|
||||
require_positive "$db_available_kb" "database available storage"
|
||||
estimated_raw_bytes=$(( (BASELINE_STREAMS + LIVE_STREAMS) * FRAMES * (PAYLOAD_BYTES + 4) * 2 ))
|
||||
required_storage_bytes=$((estimated_raw_bytes * STORAGE_MULTIPLIER))
|
||||
available_storage_bytes=$((db_available_kb * 1024))
|
||||
if [ "$available_storage_bytes" -lt "$required_storage_bytes" ]; then
|
||||
echo "database volume has $available_storage_bytes bytes free; stress preflight requires $required_storage_bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cat > "$REPORT_DIR/effective-config.json" <<EOF
|
||||
{
|
||||
"project": "$PROJECT",
|
||||
"runId": "$RUN_LABEL",
|
||||
"baselineStreams": $BASELINE_STREAMS,
|
||||
"concurrentStreams": $LIVE_STREAMS,
|
||||
"shards": $SHARDS,
|
||||
"concurrencyPerShard": $CONCURRENCY,
|
||||
"framesPerStream": $FRAMES,
|
||||
"payloadBytes": $PAYLOAD_BYTES,
|
||||
"minimumLiveDurationSeconds": $MIN_LIVE_DURATION_SECONDS,
|
||||
"maximumCaptureQueue": $MAX_CAPTURE_QUEUE,
|
||||
"captureWorkers": $CAPTURE_WORKERS,
|
||||
"captureQueueCapacity": $CAPTURE_QUEUE_CAPACITY,
|
||||
"captureBufferSizeBytes": $CAPTURE_BUFFER_SIZE_BYTES,
|
||||
"tcpTimeoutSeconds": $TCP_TIMEOUT_SECONDS,
|
||||
"udpTimeoutSeconds": $UDP_TIMEOUT_SECONDS,
|
||||
"minimumStreamsPerSecond": $MIN_STREAMS_PER_SECOND,
|
||||
"minimumCleanupStreamsPerSecond": $MIN_CLEANUP_STREAMS_PER_SECOND,
|
||||
"cleanupIntervalSeconds": $PACKMATE_NETWORK_STRESS_CLEANUP_INTERVAL_SECONDS,
|
||||
"cleanupRetentionMinutes": $PACKMATE_NETWORK_STRESS_CLEANUP_THRESHOLD_MINUTES,
|
||||
"targetIp": "$TARGET_IP",
|
||||
"databaseVolumeFreeBytesAtStart": $available_storage_bytes,
|
||||
"databaseStorageType": "$db_storage_type"
|
||||
}
|
||||
EOF
|
||||
|
||||
baseline_started=$(date +%s)
|
||||
compose run --rm --no-deps generator python /stress/api_control.py monitor \
|
||||
--stop-file /reports/baseline-load.done \
|
||||
--abort-file /reports/baseline-load.abort \
|
||||
--max-queue "$MAX_CAPTURE_QUEUE" \
|
||||
--report /reports/baseline-capture-monitor.json &
|
||||
baseline_monitor_pid=$!
|
||||
baseline_load_status=0
|
||||
if ! run_load_phase baseline "$BASELINE_STREAMS" 1 /reports/baseline-load.abort 0; then
|
||||
baseline_load_status=1
|
||||
fi
|
||||
compose exec -T reports touch /reports/baseline-load.done
|
||||
baseline_monitor_status=0
|
||||
if ! wait "$baseline_monitor_pid"; then baseline_monitor_status=1; fi
|
||||
if [ "$baseline_load_status" -ne 0 ] || [ "$baseline_monitor_status" -ne 0 ]; then
|
||||
echo "baseline traffic or continuous capture monitor failed" >&2
|
||||
exit 1
|
||||
fi
|
||||
baseline_load_elapsed=$(load_phase_elapsed baseline)
|
||||
require_positive "$baseline_load_elapsed" "baseline generator elapsed time"
|
||||
check_rate "baseline generators" "$BASELINE_STREAMS" "$baseline_load_elapsed"
|
||||
compose run --rm --no-deps generator python /stress/api_control.py metrics \
|
||||
--max-queue "$MAX_CAPTURE_QUEUE" \
|
||||
--report /reports/pcap-metrics-after-baseline-load.json
|
||||
baseline_drain_started=$(date +%s)
|
||||
wait_for_streams "end_timestamp > 0 AND favorite = false" "$BASELINE_STREAMS" "$DRAIN_TIMEOUT" "baseline network ingest"
|
||||
baseline_drain_seconds=$(( $(date +%s) - baseline_drain_started ))
|
||||
if [ "$baseline_drain_seconds" -gt "$MAX_INGEST_DRAIN_SECONDS" ]; then
|
||||
echo "baseline ingest queue needed ${baseline_drain_seconds}s to drain" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
compose run --rm --no-deps generator python /stress/api_control.py metrics \
|
||||
--require-empty-queue \
|
||||
--wait-seconds "$DRAIN_TIMEOUT" \
|
||||
--report /reports/pcap-metrics-before-cleanup.json
|
||||
|
||||
compose run --rm --no-deps generator python /stress/api_control.py verify \
|
||||
--run-id 1 \
|
||||
--sample-size 200 \
|
||||
--frames "$FRAMES" \
|
||||
--payload-bytes "$PAYLOAD_BYTES" \
|
||||
--report /reports/baseline-payload-verification.json
|
||||
|
||||
baseline_captured=$(db_integer "SELECT count(*) FROM stream")
|
||||
baseline_packets=$(db_integer "SELECT count(*) FROM packet")
|
||||
baseline_matches=$(db_integer "SELECT count(*) FROM found_pattern")
|
||||
baseline_pattern_links=$(db_integer "SELECT count(*) FROM stream_found_patterns")
|
||||
invalid_baseline_matches=$(count_invalid_matches "true")
|
||||
if [ "$baseline_captured" -ne "$BASELINE_STREAMS" ] \
|
||||
|| [ "$baseline_matches" -ne $((BASELINE_STREAMS * 2)) ] \
|
||||
|| [ "$baseline_pattern_links" -ne "$BASELINE_STREAMS" ] \
|
||||
|| [ "$invalid_baseline_matches" -ne 0 ]; then
|
||||
echo "invalid baseline counts: streams=$baseline_captured matches=$baseline_matches links=$baseline_pattern_links invalidMatches=$invalid_baseline_matches" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
verify_complete_ledger "true" 1 "$BASELINE_STREAMS" "$BASELINE_STREAMS" baseline-full-ledger.json
|
||||
|
||||
db_scalar "UPDATE stream SET favorite = true WHERE id IN (SELECT id FROM stream ORDER BY id LIMIT $FAVORITE_STREAMS)" >/dev/null
|
||||
favorite_count=$(db_integer "SELECT count(*) FROM stream WHERE favorite")
|
||||
if [ "$favorite_count" -ne "$FAVORITE_STREAMS" ]; then
|
||||
echo "could not mark $FAVORITE_STREAMS baseline streams as favorite" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
db_bytes_before=$(db_integer "SELECT pg_database_size(current_database())")
|
||||
wal_start=$(db_scalar "SELECT pg_current_wal_lsn()")
|
||||
db_scalar "UPDATE stream SET end_timestamp = 0, persisted_at = 0" >/dev/null
|
||||
|
||||
eligible_baseline=$(db_integer "SELECT count(*) FROM stream WHERE end_timestamp = 0 AND favorite = false")
|
||||
expected_eligible=$((BASELINE_STREAMS - FAVORITE_STREAMS))
|
||||
if [ "$eligible_baseline" -ne "$expected_eligible" ]; then
|
||||
echo "could not create the complete eligible baseline: eligible=$eligible_baseline expected=$expected_eligible" >&2
|
||||
exit 1
|
||||
fi
|
||||
cleanup_measurement_started_at=$(date +%s)
|
||||
printf 'unix_seconds,eligible_baseline_streams,committed_live_streams\n' > "$REPORT_DIR/cleanup-overlap.csv"
|
||||
printf '%s,%s,0\n' "$cleanup_measurement_started_at" "$eligible_baseline" >> "$REPORT_DIR/cleanup-overlap.csv"
|
||||
|
||||
PACKMATE_NETWORK_STRESS_CLEANUP_ENABLED=true
|
||||
export PACKMATE_NETWORK_STRESS_CLEANUP_ENABLED
|
||||
compose up -d --no-deps --force-recreate packmate
|
||||
compose run --rm --no-deps generator python /stress/api_control.py prepare --timeout 300
|
||||
|
||||
compose run --rm --no-deps generator python /stress/api_control.py probe \
|
||||
--requests 20000 \
|
||||
--pause 0.1 \
|
||||
--min-requests 100 \
|
||||
--stop-file /reports/workload.done \
|
||||
--abort-file /reports/live-load.abort \
|
||||
--max-queue "$MAX_CAPTURE_QUEUE" \
|
||||
--max-p99-ms "$MAX_API_P99_MS" \
|
||||
--report /reports/api-probe.json &
|
||||
probe_pid=$!
|
||||
run_load_phase live "$LIVE_STREAMS" 2 /reports/live-load.abort "$MIN_LIVE_DURATION_SECONDS" &
|
||||
live_pid=$!
|
||||
|
||||
overlap_observed=false
|
||||
cleanup_completed_at=0
|
||||
previous_old=$eligible_baseline
|
||||
previous_live=0
|
||||
while kill -0 "$live_pid" 2>/dev/null; do
|
||||
sample_time=$(date +%s)
|
||||
remaining_old=$(db_integer "SELECT count(*) FROM stream WHERE end_timestamp = 0 AND favorite = false")
|
||||
committed_live=$(db_integer "SELECT count(*) FROM stream WHERE end_timestamp > 0 AND favorite = false")
|
||||
printf '%s,%s,%s\n' "$sample_time" "$remaining_old" "$committed_live" >> "$REPORT_DIR/cleanup-overlap.csv"
|
||||
if [ "$remaining_old" -lt "$previous_old" ] \
|
||||
&& [ "$committed_live" -gt "$previous_live" ] \
|
||||
&& [ "$previous_live" -gt 0 ]; then
|
||||
overlap_observed=true
|
||||
fi
|
||||
if [ "$remaining_old" -eq 0 ] && [ "$cleanup_completed_at" -eq 0 ]; then
|
||||
cleanup_completed_at=$sample_time
|
||||
fi
|
||||
if ! kill -0 "$probe_pid" 2>/dev/null; then
|
||||
echo "API/capture probe exited while concurrent traffic was active" >&2
|
||||
compose exec -T reports touch /reports/live-load.abort
|
||||
fi
|
||||
previous_old=$remaining_old
|
||||
previous_live=$committed_live
|
||||
sleep 1
|
||||
done
|
||||
|
||||
live_load_status=0
|
||||
if ! wait "$live_pid"; then live_load_status=1; fi
|
||||
if [ "$live_load_status" -ne 0 ]; then
|
||||
compose exec -T reports touch /reports/workload.done
|
||||
wait "$probe_pid" || true
|
||||
echo "concurrent traffic was aborted or a generator failed" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$overlap_observed" != true ]; then
|
||||
compose exec -T reports touch /reports/workload.done
|
||||
wait "$probe_pid" || true
|
||||
echo "cleanup did not overlap committed live network traffic" >&2
|
||||
exit 1
|
||||
fi
|
||||
live_load_elapsed=$(load_phase_elapsed live)
|
||||
require_positive "$live_load_elapsed" "live generator elapsed time"
|
||||
check_rate "live generators" "$LIVE_STREAMS" "$live_load_elapsed"
|
||||
live_drain_started=$(date +%s)
|
||||
wait_for_streams "end_timestamp > 0 AND favorite = false" "$LIVE_STREAMS" "$DRAIN_TIMEOUT" "concurrent network ingest" "$probe_pid"
|
||||
live_drain_seconds=$(( $(date +%s) - live_drain_started ))
|
||||
if [ "$live_drain_seconds" -gt "$MAX_INGEST_DRAIN_SECONDS" ]; then
|
||||
echo "live ingest queue needed ${live_drain_seconds}s to drain" >&2
|
||||
exit 1
|
||||
fi
|
||||
wait_for_zero "end_timestamp = 0 AND favorite = false" "$CLEANUP_TIMEOUT" "eligible cleanup backlog" "$probe_pid"
|
||||
if [ "$cleanup_completed_at" -eq 0 ]; then cleanup_completed_at=$(date +%s); fi
|
||||
cleanup_elapsed_seconds=$((cleanup_completed_at - cleanup_measurement_started_at))
|
||||
if [ "$cleanup_elapsed_seconds" -lt 1 ]; then cleanup_elapsed_seconds=1; fi
|
||||
cleanup_streams_per_second=$((eligible_baseline / cleanup_elapsed_seconds))
|
||||
if [ "$cleanup_streams_per_second" -lt "$MIN_CLEANUP_STREAMS_PER_SECOND" ]; then
|
||||
echo "cleanup sustained only $cleanup_streams_per_second streams/s; required $MIN_CLEANUP_STREAMS_PER_SECOND" >&2
|
||||
exit 1
|
||||
fi
|
||||
compose run --rm -T --no-deps generator sh -c 'touch /reports/workload.done'
|
||||
wait "$probe_pid"
|
||||
|
||||
compose run --rm --no-deps generator python /stress/api_control.py verify \
|
||||
--run-id 2 \
|
||||
--sample-size 200 \
|
||||
--frames "$FRAMES" \
|
||||
--payload-bytes "$PAYLOAD_BYTES" \
|
||||
--report /reports/concurrent-payload-verification.json
|
||||
|
||||
verify_complete_ledger "s.favorite = false" 2 "$LIVE_STREAMS" "$LIVE_STREAMS" concurrent-full-ledger.json
|
||||
verify_complete_ledger "s.favorite = true" 1 "$BASELINE_STREAMS" "$FAVORITE_STREAMS" favorite-full-ledger.json true
|
||||
|
||||
compose run --rm --no-deps generator python /stress/api_control.py metrics \
|
||||
--require-empty-queue \
|
||||
--wait-seconds "$DRAIN_TIMEOUT" \
|
||||
--report /reports/pcap-metrics.json
|
||||
|
||||
orphan_packets=$(db_integer "SELECT count(*) FROM packet p LEFT JOIN stream s ON s.id=p.stream_id WHERE s.id IS NULL")
|
||||
orphan_matches=$(db_integer "SELECT count(*) FROM found_pattern fp LEFT JOIN packet p ON p.id=fp.packet_id WHERE p.id IS NULL")
|
||||
orphan_pattern_links=$(db_integer "SELECT count(*) FROM stream_found_patterns sfp LEFT JOIN stream s ON s.id=sfp.stream_id LEFT JOIN pattern pt ON pt.id=sfp.pattern_id WHERE s.id IS NULL OR pt.id IS NULL")
|
||||
cleanup_pending=$(db_integer "SELECT count(*) FROM stream WHERE cleanup_pending")
|
||||
final_matches=$(db_integer "SELECT count(*) FROM found_pattern")
|
||||
final_pattern_links=$(db_integer "SELECT count(*) FROM stream_found_patterns")
|
||||
invalid_final_matches=$(count_invalid_matches "true")
|
||||
if [ "$orphan_packets" -ne 0 ] \
|
||||
|| [ "$orphan_matches" -ne 0 ] \
|
||||
|| [ "$orphan_pattern_links" -ne 0 ] \
|
||||
|| [ "$cleanup_pending" -ne 0 ] \
|
||||
|| [ "$final_matches" -ne $(((LIVE_STREAMS + FAVORITE_STREAMS) * 2)) ] \
|
||||
|| [ "$final_pattern_links" -ne $((LIVE_STREAMS + FAVORITE_STREAMS)) ] \
|
||||
|| [ "$invalid_final_matches" -ne 0 ]; then
|
||||
echo "invalid final state: orphan_packets=$orphan_packets orphan_matches=$orphan_matches orphan_links=$orphan_pattern_links cleanup_pending=$cleanup_pending matches=$final_matches links=$final_pattern_links invalidMatches=$invalid_final_matches" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
live_captured=$(db_integer "SELECT count(*) FROM stream WHERE end_timestamp > 0 AND favorite = false")
|
||||
favorite_captured=$(db_integer "SELECT count(*) FROM stream WHERE favorite")
|
||||
final_packets=$(db_integer "SELECT count(*) FROM packet")
|
||||
db_bytes_after=$(db_integer "SELECT pg_database_size(current_database())")
|
||||
wal_bytes=$(db_integer "SELECT pg_wal_lsn_diff(pg_current_wal_lsn(), '$wal_start')::bigint")
|
||||
elapsed=$(( $(date +%s) - baseline_started ))
|
||||
|
||||
cat > "$REPORT_DIR/summary.json" <<EOF
|
||||
{
|
||||
"baselineRequestedStreams": $BASELINE_STREAMS,
|
||||
"baselineCapturedStreams": $baseline_captured,
|
||||
"baselineCapturedPackets": $baseline_packets,
|
||||
"baselineMatches": $baseline_matches,
|
||||
"baselinePatternLinks": $baseline_pattern_links,
|
||||
"concurrentRequestedStreams": $LIVE_STREAMS,
|
||||
"concurrentCapturedStreams": $live_captured,
|
||||
"favoriteStreamsPreserved": $favorite_captured,
|
||||
"finalPackets": $final_packets,
|
||||
"finalMatches": $final_matches,
|
||||
"finalPatternLinks": $final_pattern_links,
|
||||
"shards": $SHARDS,
|
||||
"framesPerStream": $FRAMES,
|
||||
"payloadBytes": $PAYLOAD_BYTES,
|
||||
"captureWorkers": $CAPTURE_WORKERS,
|
||||
"captureQueueCapacity": $CAPTURE_QUEUE_CAPACITY,
|
||||
"captureBufferSizeBytes": $CAPTURE_BUFFER_SIZE_BYTES,
|
||||
"tcpTimeoutSeconds": $TCP_TIMEOUT_SECONDS,
|
||||
"elapsedSeconds": $elapsed,
|
||||
"baselineLoadSeconds": $baseline_load_elapsed,
|
||||
"baselineIngestDrainSeconds": $baseline_drain_seconds,
|
||||
"liveLoadSeconds": $live_load_elapsed,
|
||||
"liveIngestDrainSeconds": $live_drain_seconds,
|
||||
"minimumStreamsPerSecond": $MIN_STREAMS_PER_SECOND,
|
||||
"cleanupOverlapObserved": $overlap_observed,
|
||||
"cleanupElapsedSeconds": $cleanup_elapsed_seconds,
|
||||
"cleanupStreamsPerSecond": $cleanup_streams_per_second,
|
||||
"minimumCleanupStreamsPerSecond": $MIN_CLEANUP_STREAMS_PER_SECOND,
|
||||
"walBytesDuringCleanup": $wal_bytes,
|
||||
"databaseBytesBeforeCleanup": $db_bytes_before,
|
||||
"databaseBytesAfterCleanup": $db_bytes_after,
|
||||
"databaseVolumeFreeBytesAtStart": $available_storage_bytes,
|
||||
"databaseStorageType": "$db_storage_type",
|
||||
"estimatedRawTrafficBytes": $estimated_raw_bytes,
|
||||
"requiredStoragePreflightBytes": $required_storage_bytes,
|
||||
"orphanPackets": $orphan_packets,
|
||||
"orphanMatches": $orphan_matches,
|
||||
"orphanPatternLinks": $orphan_pattern_links,
|
||||
"cleanupPending": $cleanup_pending
|
||||
}
|
||||
EOF
|
||||
|
||||
cat "$REPORT_DIR/summary.json"
|
||||
echo "Network capture and cleanup stress PASSED"
|
||||
40
stress/network/tcp_target.py
Normal file
@ -0,0 +1,40 @@
|
||||
import asyncio
|
||||
import os
|
||||
import struct
|
||||
|
||||
|
||||
HOST = os.getenv("STRESS_TARGET_HOST", "0.0.0.0")
|
||||
PORT = int(os.getenv("STRESS_TARGET_PORT", "18080"))
|
||||
MAX_FRAME_BYTES = 1_048_576
|
||||
|
||||
|
||||
async def handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
|
||||
try:
|
||||
while True:
|
||||
header = await reader.readexactly(4)
|
||||
size = struct.unpack("!I", header)[0]
|
||||
if size <= 0 or size > MAX_FRAME_BYTES:
|
||||
raise ValueError(f"invalid frame size: {size}")
|
||||
payload = await reader.readexactly(size)
|
||||
writer.write(struct.pack("!I", len(payload)) + payload)
|
||||
await writer.drain()
|
||||
except (asyncio.IncompleteReadError, ConnectionError, ValueError):
|
||||
pass
|
||||
finally:
|
||||
writer.close()
|
||||
try:
|
||||
await writer.wait_closed()
|
||||
except ConnectionError:
|
||||
pass
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
server = await asyncio.start_server(handle, HOST, PORT, backlog=4096, limit=2 * MAX_FRAME_BYTES)
|
||||
addresses = ", ".join(str(sock.getsockname()) for sock in server.sockets or [])
|
||||
print(f"TCP stress target listening on {addresses}", flush=True)
|
||||
async with server:
|
||||
await server.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
170
stress/network/verify_ledger.py
Normal file
@ -0,0 +1,170 @@
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PAYLOAD_MARKER = b"PMSTRESS"
|
||||
PAD_CYCLE = bytes(range(251))
|
||||
|
||||
|
||||
def build_payload(run_id: int, connection: int, shard: int, sequence: int, payload_bytes: int) -> bytes:
|
||||
prefix = struct.pack("!QQII", run_id, connection, shard, sequence)
|
||||
padding_size = payload_bytes - len(prefix)
|
||||
start = (connection + sequence) % len(PAD_CYCLE)
|
||||
rotated = PAD_CYCLE[start:] + PAD_CYCLE[:start]
|
||||
padding = bytearray((rotated * ((padding_size + len(PAD_CYCLE) - 1) // len(PAD_CYCLE)))[:padding_size])
|
||||
if sequence == 0:
|
||||
padding[:len(PAYLOAD_MARKER)] = PAYLOAD_MARKER
|
||||
return prefix + padding
|
||||
|
||||
|
||||
def expected_digest(
|
||||
run_id: int,
|
||||
connection: int,
|
||||
shard: int,
|
||||
frames: int,
|
||||
payload_bytes: int,
|
||||
) -> str:
|
||||
digest = hashlib.md5()
|
||||
for sequence in range(frames):
|
||||
payload = build_payload(run_id, connection, shard, sequence, payload_bytes)
|
||||
digest.update(struct.pack("!I", len(payload)))
|
||||
digest.update(payload)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def expected_shard(connection: int, streams: int, shards: int) -> int:
|
||||
base, remainder = divmod(streams, shards)
|
||||
larger_range = (base + 1) * remainder
|
||||
if connection < larger_range:
|
||||
return connection // (base + 1)
|
||||
if base == 0:
|
||||
raise ValueError(f"connection {connection} is outside non-empty shards")
|
||||
return remainder + (connection - larger_range) // base
|
||||
|
||||
|
||||
def verify(args: argparse.Namespace) -> dict:
|
||||
errors: list[str] = []
|
||||
seen_connections: set[int] = set()
|
||||
rows = 0
|
||||
valid_rows = 0
|
||||
expected_length = args.frames * (args.payload_bytes + 4)
|
||||
|
||||
def fail(message: str) -> None:
|
||||
if len(errors) < 100:
|
||||
errors.append(message)
|
||||
|
||||
for raw_line in sys.stdin:
|
||||
line = raw_line.rstrip("\r\n")
|
||||
if not line:
|
||||
continue
|
||||
rows += 1
|
||||
stream_id = "?"
|
||||
try:
|
||||
(
|
||||
stream_id_text,
|
||||
marker_hex,
|
||||
incoming_length_text,
|
||||
incoming_hash,
|
||||
outgoing_length_text,
|
||||
outgoing_hash,
|
||||
ips_valid_text,
|
||||
metadata_valid_text,
|
||||
) = line.split("\t")
|
||||
stream_id = int(stream_id_text)
|
||||
marker = bytes.fromhex(marker_hex)
|
||||
if len(marker) != 24:
|
||||
raise ValueError(f"marker has {len(marker)} bytes")
|
||||
run_id, connection, shard, sequence = struct.unpack("!QQII", marker)
|
||||
if run_id != args.run_id or sequence != 0:
|
||||
raise ValueError(f"marker run={run_id} sequence={sequence}")
|
||||
if connection >= args.total_streams:
|
||||
raise ValueError(f"connection {connection} is out of range")
|
||||
required_shard = expected_shard(connection, args.total_streams, args.shards)
|
||||
if shard != required_shard:
|
||||
raise ValueError(f"connection {connection} has shard={shard}, expected={required_shard}")
|
||||
if connection in seen_connections:
|
||||
raise ValueError(f"connection {connection} appears in more than one captured stream")
|
||||
incoming_length = int(incoming_length_text)
|
||||
outgoing_length = int(outgoing_length_text)
|
||||
if incoming_length != expected_length or outgoing_length != expected_length:
|
||||
raise ValueError(
|
||||
f"payload lengths incoming={incoming_length} outgoing={outgoing_length} "
|
||||
f"expected={expected_length}"
|
||||
)
|
||||
digest = expected_digest(run_id, connection, shard, args.frames, args.payload_bytes)
|
||||
if incoming_hash != digest or outgoing_hash != digest:
|
||||
raise ValueError(
|
||||
f"payload hash incoming={incoming_hash} outgoing={outgoing_hash} expected={digest}"
|
||||
)
|
||||
if ips_valid_text != "t":
|
||||
raise ValueError("packet IP directions are invalid")
|
||||
if metadata_valid_text != "t":
|
||||
raise ValueError("stream packet count, byte size, service, protocol or timestamps are invalid")
|
||||
seen_connections.add(connection)
|
||||
valid_rows += 1
|
||||
except (ValueError, struct.error) as exc:
|
||||
fail(f"row={rows} stream={stream_id}: {exc}")
|
||||
|
||||
if rows != args.expected_streams:
|
||||
fail(f"captured ledger contains {rows} streams, expected {args.expected_streams}")
|
||||
if not args.allow_subset and len(seen_connections) != args.total_streams:
|
||||
missing = []
|
||||
for connection in range(args.total_streams):
|
||||
if connection not in seen_connections:
|
||||
missing.append(connection)
|
||||
if len(missing) == 20:
|
||||
break
|
||||
fail(
|
||||
f"captured unique connections={len(seen_connections)}, expected={args.total_streams}, "
|
||||
f"sampleMissing={missing}"
|
||||
)
|
||||
|
||||
result = {
|
||||
"runId": args.run_id,
|
||||
"ledgerRows": rows,
|
||||
"validRows": valid_rows,
|
||||
"uniqueConnections": len(seen_connections),
|
||||
"expectedStreams": args.expected_streams,
|
||||
"totalOfferedStreams": args.total_streams,
|
||||
"allowSubset": args.allow_subset,
|
||||
"failedRows": rows - valid_rows,
|
||||
"sampleErrors": errors,
|
||||
}
|
||||
Path(args.report).write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(result), flush=True)
|
||||
return result
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Verify every captured connection from a PostgreSQL TSV ledger")
|
||||
parser.add_argument("--run-id", type=int, required=True)
|
||||
parser.add_argument("--total-streams", type=int, required=True)
|
||||
parser.add_argument("--expected-streams", type=int, required=True)
|
||||
parser.add_argument("--shards", type=int, required=True)
|
||||
parser.add_argument("--frames", type=int, required=True)
|
||||
parser.add_argument("--payload-bytes", type=int, required=True)
|
||||
parser.add_argument("--allow-subset", action="store_true")
|
||||
parser.add_argument("--report", required=True)
|
||||
args = parser.parse_args()
|
||||
if min(
|
||||
args.run_id,
|
||||
args.total_streams,
|
||||
args.expected_streams,
|
||||
args.shards,
|
||||
args.frames,
|
||||
args.payload_bytes,
|
||||
) <= 0:
|
||||
parser.error("all numeric arguments must be positive")
|
||||
if args.expected_streams > args.total_streams:
|
||||
parser.error("expected streams cannot exceed total offered streams")
|
||||
return args
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
arguments = parse_args()
|
||||
result = verify(arguments)
|
||||
raise SystemExit(0 if not result["sampleErrors"] else 1)
|
||||