180 lines
6.8 KiB
Python
180 lines
6.8 KiB
Python
"""Index and read Medulla DObject recordings (dobject/ + dobject_recording/)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import struct
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import BinaryIO, Iterator
|
|
|
|
|
|
RECORD_RE = re.compile(
|
|
r"^\[(?P<log_time>[^]]+)\].*?DObject `(?P<name>[^`]+)` post "
|
|
r"len=(?P<len>\d+)B, id:(?P<id>[0-9A-Fa-f]+), tic:(?P<tic>\d+), "
|
|
r"@(?P<file>[^:]+):(?P<offset>\d+)"
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RecordRef:
|
|
sequence: int
|
|
object_name: str
|
|
log_time: str
|
|
source_log: str
|
|
source_dorec: str
|
|
source_offset: int
|
|
payload_length: int
|
|
log_record_id: str
|
|
dotnet_ticks: int
|
|
|
|
|
|
def resolve_dlog_root(value: Path | str) -> Path:
|
|
root = Path(value).expanduser().resolve()
|
|
if (root / "dobject").is_dir() and (root / "dobject_recording").is_dir():
|
|
return root
|
|
child = root / "dlog"
|
|
if (child / "dobject").is_dir() and (child / "dobject_recording").is_dir():
|
|
return child
|
|
raise FileNotFoundError(f"{root} does not contain dobject and dobject_recording")
|
|
|
|
|
|
def discover_records(dlog_root: Path, object_name: str) -> list[RecordRef]:
|
|
pending: list[tuple[str, str, str, int, int, str, int, str]] = []
|
|
for log_path in sorted((dlog_root / "dobject").rglob("*.log")):
|
|
relative_log = log_path.relative_to(dlog_root).as_posix()
|
|
with log_path.open("r", encoding="utf-8", errors="replace") as stream:
|
|
for line in stream:
|
|
match = RECORD_RE.search(line)
|
|
if not match or match.group("name").casefold() != object_name.casefold():
|
|
continue
|
|
pending.append(
|
|
(
|
|
match.group("name"),
|
|
match.group("log_time"),
|
|
relative_log,
|
|
int(match.group("offset")),
|
|
int(match.group("len")),
|
|
match.group("id").upper(),
|
|
int(match.group("tic")),
|
|
match.group("file"),
|
|
)
|
|
)
|
|
pending.sort(key=lambda item: (item[6], item[7].casefold(), item[3]))
|
|
seen: set[tuple[str, int, int]] = set()
|
|
records: list[RecordRef] = []
|
|
for item in pending:
|
|
key = (item[7].casefold(), item[3], item[6])
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
records.append(
|
|
RecordRef(
|
|
sequence=len(records),
|
|
object_name=item[0],
|
|
log_time=item[1],
|
|
source_log=item[2],
|
|
source_dorec=item[7],
|
|
source_offset=item[3],
|
|
payload_length=item[4],
|
|
log_record_id=item[5],
|
|
dotnet_ticks=item[6],
|
|
)
|
|
)
|
|
return records
|
|
|
|
|
|
def index_dorec_files(dlog_root: Path) -> dict[str, list[Path]]:
|
|
result: dict[str, list[Path]] = {}
|
|
for path in (dlog_root / "dobject_recording").rglob("*.dorec"):
|
|
result.setdefault(path.name.casefold(), []).append(path)
|
|
return result
|
|
|
|
|
|
def choose_dorec(index: dict[str, list[Path]], name: str) -> Path:
|
|
matches = index.get(Path(name).name.casefold(), [])
|
|
if not matches:
|
|
raise FileNotFoundError(f"missing recording file: {name}")
|
|
if len(matches) > 1:
|
|
raise RuntimeError(f"ambiguous recording file {name}: {matches}")
|
|
return matches[0]
|
|
|
|
|
|
def read_exact(stream: BinaryIO, size: int) -> bytes:
|
|
data = stream.read(size)
|
|
if len(data) != size:
|
|
raise EOFError(f"expected {size} bytes, got {len(data)}")
|
|
return data
|
|
|
|
|
|
def read_record_payload(path: Path, record: RecordRef) -> bytes:
|
|
with path.open("rb") as stream:
|
|
stream.seek(record.source_offset)
|
|
name_length = read_exact(stream, 1)[0]
|
|
name = read_exact(stream, name_length).decode("ascii")
|
|
ticks = struct.unpack("<q", read_exact(stream, 8))[0]
|
|
id_length = read_exact(stream, 1)[0]
|
|
id_bytes = read_exact(stream, id_length)
|
|
payload_length = struct.unpack("<i", read_exact(stream, 4))[0]
|
|
payload = read_exact(stream, payload_length)
|
|
|
|
try:
|
|
record_id = id_bytes.decode("ascii")
|
|
except UnicodeDecodeError:
|
|
record_id = id_bytes.hex().upper()
|
|
if name != record.object_name:
|
|
raise ValueError(f"name mismatch: log={record.object_name}, dorec={name}")
|
|
if ticks != record.dotnet_ticks:
|
|
raise ValueError(f"tick mismatch: log={record.dotnet_ticks}, dorec={ticks}")
|
|
if payload_length != record.payload_length:
|
|
raise ValueError(f"payload mismatch: log={record.payload_length}, dorec={payload_length}")
|
|
if record_id.upper() != record.log_record_id.upper():
|
|
raise ValueError(f"record id mismatch: log={record.log_record_id}, dorec={record_id}")
|
|
return payload
|
|
|
|
|
|
def iter_payloads(dlog_root: Path, object_name: str) -> Iterator[tuple[RecordRef, bytes]]:
|
|
root = resolve_dlog_root(dlog_root)
|
|
records = discover_records(root, object_name)
|
|
if not records:
|
|
return
|
|
dorec_index = index_dorec_files(root)
|
|
open_files: dict[str, tuple[Path, BinaryIO]] = {}
|
|
try:
|
|
for record in records:
|
|
key = record.source_dorec.casefold()
|
|
handle = open_files.get(key)
|
|
if handle is None:
|
|
path = choose_dorec(dorec_index, record.source_dorec)
|
|
handle = (path, path.open("rb"))
|
|
open_files[key] = handle
|
|
path, stream = handle
|
|
stream.seek(record.source_offset)
|
|
name_length = read_exact(stream, 1)[0]
|
|
name = read_exact(stream, name_length).decode("ascii")
|
|
ticks = struct.unpack("<q", read_exact(stream, 8))[0]
|
|
id_length = read_exact(stream, 1)[0]
|
|
id_bytes = read_exact(stream, id_length)
|
|
payload_length = struct.unpack("<i", read_exact(stream, 4))[0]
|
|
payload = read_exact(stream, payload_length)
|
|
try:
|
|
record_id = id_bytes.decode("ascii")
|
|
except UnicodeDecodeError:
|
|
record_id = id_bytes.hex().upper()
|
|
if name != record.object_name:
|
|
raise ValueError(f"name mismatch: log={record.object_name}, dorec={name}")
|
|
if ticks != record.dotnet_ticks:
|
|
raise ValueError(f"tick mismatch: log={record.dotnet_ticks}, dorec={ticks}")
|
|
if payload_length != record.payload_length:
|
|
raise ValueError(
|
|
f"payload mismatch: log={record.payload_length}, dorec={payload_length}"
|
|
)
|
|
if record_id.upper() != record.log_record_id.upper():
|
|
raise ValueError(
|
|
f"record id mismatch: log={record.log_record_id}, dorec={record_id}"
|
|
)
|
|
yield record, payload
|
|
finally:
|
|
for _path, stream in open_files.values():
|
|
stream.close()
|