255 lines
7.9 KiB
Python
255 lines
7.9 KiB
Python
from __future__ import annotations
|
|
|
|
import binascii
|
|
import io
|
|
import struct
|
|
from dataclasses import dataclass, asdict
|
|
from pathlib import Path
|
|
from typing import BinaryIO, Iterator
|
|
|
|
|
|
FILE_MAGIC = "RAW_SERIAL_CAPTURE_FILE_V2"
|
|
RECORD_MAGIC = "RAW_SERIAL_RECORD_V2"
|
|
FOOTER_MAGIC = "RAW_SERIAL_CAPTURE_FOOTER_V2"
|
|
|
|
|
|
def read_7bit_int(stream: BinaryIO) -> int:
|
|
value = 0
|
|
shift = 0
|
|
while True:
|
|
raw = stream.read(1)
|
|
if not raw:
|
|
raise EOFError("truncated .NET string length")
|
|
value |= (raw[0] & 0x7F) << shift
|
|
if not raw[0] & 0x80:
|
|
return value
|
|
shift += 7
|
|
if shift > 35:
|
|
raise ValueError("invalid .NET string length")
|
|
|
|
|
|
def read_dotnet_string(stream: BinaryIO) -> str:
|
|
length = read_7bit_int(stream)
|
|
raw = stream.read(length)
|
|
if len(raw) != length:
|
|
raise EOFError("truncated .NET string")
|
|
return raw.decode("utf-8")
|
|
|
|
|
|
def read_i32(stream: BinaryIO) -> int:
|
|
raw = stream.read(4)
|
|
if len(raw) != 4:
|
|
raise EOFError("truncated int32")
|
|
return struct.unpack("<i", raw)[0]
|
|
|
|
|
|
def read_i64(stream: BinaryIO) -> int:
|
|
raw = stream.read(8)
|
|
if len(raw) != 8:
|
|
raise EOFError("truncated int64")
|
|
return struct.unpack("<q", raw)[0]
|
|
|
|
|
|
def read_u32(stream: BinaryIO) -> int:
|
|
raw = stream.read(4)
|
|
if len(raw) != 4:
|
|
raise EOFError("truncated uint32")
|
|
return struct.unpack("<I", raw)[0]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CaptureHeader:
|
|
sensor_kind: str
|
|
session_id: str
|
|
session_start_utc_ticks: int
|
|
session_start_monotonic_ticks: int
|
|
monotonic_frequency: int
|
|
port: str
|
|
baud: int
|
|
file_start_utc_ticks: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RawChunk:
|
|
sequence: int
|
|
receive_utc_ticks: int
|
|
receive_monotonic_ticks: int
|
|
raw: bytes
|
|
record_file_offset: int
|
|
raw_file_offset: int
|
|
record_crc32: int
|
|
crc_valid: bool
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CaptureFooter:
|
|
clean_close: bool
|
|
records: int
|
|
bytes: int
|
|
first_sequence: int
|
|
last_sequence: int
|
|
dropped_chunks: int
|
|
dropped_bytes: int
|
|
crc_valid: bool
|
|
|
|
|
|
@dataclass
|
|
class CaptureFile:
|
|
path: str
|
|
header: CaptureHeader
|
|
chunks: list[RawChunk]
|
|
footer: CaptureFooter | None
|
|
truncated_tail: bool = False
|
|
|
|
|
|
def read_header(stream: BinaryIO) -> CaptureHeader:
|
|
if read_dotnet_string(stream) != FILE_MAGIC:
|
|
raise ValueError("not a V2 raw capture file")
|
|
version = read_i32(stream)
|
|
if version != 2:
|
|
raise ValueError(f"unsupported capture version: {version}")
|
|
return CaptureHeader(
|
|
sensor_kind=read_dotnet_string(stream),
|
|
session_id=read_dotnet_string(stream),
|
|
session_start_utc_ticks=read_i64(stream),
|
|
session_start_monotonic_ticks=read_i64(stream),
|
|
monotonic_frequency=read_i64(stream),
|
|
port=read_dotnet_string(stream),
|
|
baud=read_i32(stream),
|
|
file_start_utc_ticks=read_i64(stream),
|
|
)
|
|
|
|
|
|
def parse_record_body(body: bytes, record_file_offset: int, record_crc: int) -> RawChunk:
|
|
stream = io.BytesIO(body)
|
|
if read_dotnet_string(stream) != RECORD_MAGIC:
|
|
raise ValueError("invalid record magic")
|
|
sequence = read_i64(stream)
|
|
receive_utc_ticks = read_i64(stream)
|
|
receive_monotonic_ticks = read_i64(stream)
|
|
raw_length = read_i32(stream)
|
|
if raw_length < 0 or raw_length > 64 * 1024 * 1024:
|
|
raise ValueError(f"invalid raw length: {raw_length}")
|
|
raw_offset = record_file_offset + 4 + stream.tell()
|
|
raw = stream.read(raw_length)
|
|
if len(raw) != raw_length:
|
|
raise EOFError("truncated raw bytes")
|
|
crc_valid = (binascii.crc32(body) & 0xFFFFFFFF) == record_crc
|
|
return RawChunk(
|
|
sequence=sequence,
|
|
receive_utc_ticks=receive_utc_ticks,
|
|
receive_monotonic_ticks=receive_monotonic_ticks,
|
|
raw=raw,
|
|
record_file_offset=record_file_offset,
|
|
raw_file_offset=raw_offset,
|
|
record_crc32=record_crc,
|
|
crc_valid=crc_valid,
|
|
)
|
|
|
|
|
|
def parse_footer(body: bytes, expected_crc: int) -> CaptureFooter:
|
|
stream = io.BytesIO(body)
|
|
if read_dotnet_string(stream) != FOOTER_MAGIC:
|
|
raise ValueError("invalid footer magic")
|
|
clean_close = stream.read(1) == b"\x01"
|
|
records = read_i64(stream)
|
|
raw_bytes = read_i64(stream)
|
|
first_sequence = read_i64(stream)
|
|
last_sequence = read_i64(stream)
|
|
dropped_chunks = read_i64(stream)
|
|
dropped_bytes = read_i64(stream)
|
|
return CaptureFooter(
|
|
clean_close=clean_close,
|
|
records=records,
|
|
bytes=raw_bytes,
|
|
first_sequence=first_sequence,
|
|
last_sequence=last_sequence,
|
|
dropped_chunks=dropped_chunks,
|
|
dropped_bytes=dropped_bytes,
|
|
crc_valid=(binascii.crc32(body) & 0xFFFFFFFF) == expected_crc,
|
|
)
|
|
|
|
|
|
def read_capture(path: Path) -> CaptureFile:
|
|
chunks: list[RawChunk] = []
|
|
footer = None
|
|
truncated = False
|
|
with path.open("rb") as stream:
|
|
header = read_header(stream)
|
|
while True:
|
|
record_offset = stream.tell()
|
|
length_raw = stream.read(4)
|
|
if not length_raw:
|
|
break
|
|
if len(length_raw) != 4:
|
|
truncated = True
|
|
break
|
|
length = struct.unpack("<i", length_raw)[0]
|
|
try:
|
|
if length == -1:
|
|
footer_length = read_i32(stream)
|
|
if footer_length < 0 or footer_length > 1024 * 1024:
|
|
raise ValueError("invalid footer length")
|
|
footer_body = stream.read(footer_length)
|
|
if len(footer_body) != footer_length:
|
|
raise EOFError("truncated footer")
|
|
footer = parse_footer(footer_body, read_u32(stream))
|
|
break
|
|
if length <= 0 or length > 64 * 1024 * 1024:
|
|
raise ValueError("invalid record length")
|
|
body = stream.read(length)
|
|
if len(body) != length:
|
|
raise EOFError("truncated record body")
|
|
record_crc = read_u32(stream)
|
|
chunks.append(parse_record_body(body, record_offset, record_crc))
|
|
except (EOFError, ValueError):
|
|
truncated = True
|
|
break
|
|
return CaptureFile(str(path), header, chunks, footer, truncated)
|
|
|
|
|
|
def sequence_gaps(chunks: list[RawChunk]) -> list[tuple[int, int, int]]:
|
|
result = []
|
|
for previous, current in zip(chunks, chunks[1:]):
|
|
if current.sequence > previous.sequence + 1:
|
|
result.append((previous.sequence, current.sequence, current.sequence - previous.sequence - 1))
|
|
return result
|
|
|
|
|
|
def file_summary(capture: CaptureFile) -> dict:
|
|
gaps = sequence_gaps(capture.chunks)
|
|
sequences = [chunk.sequence for chunk in capture.chunks]
|
|
return {
|
|
"path": capture.path,
|
|
"sensor": capture.header.sensor_kind,
|
|
"session_id": capture.header.session_id,
|
|
"port": capture.header.port,
|
|
"baud": capture.header.baud,
|
|
"chunks_read": len(capture.chunks),
|
|
"bytes_read": sum(len(chunk.raw) for chunk in capture.chunks),
|
|
"first_sequence": sequences[0] if sequences else None,
|
|
"last_sequence": sequences[-1] if sequences else None,
|
|
"missing_chunks": sum(gap[2] for gap in gaps),
|
|
"gap_count": len(gaps),
|
|
"bad_record_crc": sum(not chunk.crc_valid for chunk in capture.chunks),
|
|
"truncated_tail": capture.truncated_tail,
|
|
"footer": None if capture.footer is None else asdict(capture.footer),
|
|
"gaps": gaps[:100],
|
|
}
|
|
|
|
|
|
def iter_contiguous_segments(chunks: list[RawChunk]) -> Iterator[tuple[int, list[RawChunk]]]:
|
|
if not chunks:
|
|
return
|
|
segment_id = 0
|
|
current = [chunks[0]]
|
|
for previous, chunk in zip(chunks, chunks[1:]):
|
|
if chunk.sequence != previous.sequence + 1:
|
|
yield segment_id, current
|
|
segment_id += 1
|
|
current = [chunk]
|
|
else:
|
|
current.append(chunk)
|
|
yield segment_id, current
|
|
|