sjg/trx-rs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[fix](trx-frontend-http): group decode history by decoder

Browse Source 
Serve grouped decode history payloads and restore each decoder through
explicit history restore hooks instead of replaying a mixed message stream.

This reduces replay overhead further by removing type regrouping and keeping
history restoration on decoder-specific bulk paths.

Co-authored-by: OpenAI Codex <codex@openai.com>
Signed-off-by: Stan Grams <sjg@haxx.space>
This commit is contained in:
Stan Grams
2026-03-14 18:11:09 +01:00
parent cba3751f0e
commit 73b1d5618d
10 changed files with 176 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/trx-client/trx-frontend/trx-frontend-http/assets/web/app.js
+80 -68
View File
@@ -7361,9 +7361,8 @@ function dispatchDecodeBatch(batch) {
} }
} }
const DECODE_HISTORY_MAX_BATCH = 256;
const DECODE_HISTORY_TYPE_BATCH_LIMIT = 192; const DECODE_HISTORY_TYPE_BATCH_LIMIT = 192;
const DECODE_HISTORY_SLICE_BUDGET_MS = 10; const DECODE_HISTORY_WORKER_GROUP_LIMIT = 512;
const DECODE_HISTORY_BATCH_DRAIN_BUDGET_MS = 8; const DECODE_HISTORY_BATCH_DRAIN_BUDGET_MS = 8;
function terminateDecodeHistoryWorker() { function terminateDecodeHistoryWorker() {
@@ -7381,52 +7380,53 @@ function scheduleDecodeHistoryDrainStep(callback) {
} }
} }
function drainDecodeHistory(buffer, index, onDone, onProgress) {
const startedAt = typeof performance !== "undefined" && typeof performance.now === "function"
? performance.now()
: 0;
let nextIndex = index;
while (nextIndex < buffer.length) {
const batchStart = nextIndex;
const batchType = String(buffer[nextIndex]?.type || "");
nextIndex += 1;
while (
nextIndex < buffer.length
&& (nextIndex - batchStart) < DECODE_HISTORY_TYPE_BATCH_LIMIT
&& (nextIndex - index) < DECODE_HISTORY_MAX_BATCH
&& String(buffer[nextIndex]?.type || "") === batchType
) {
nextIndex += 1;
}
dispatchDecodeBatch(buffer.slice(batchStart, nextIndex));
if ((nextIndex - index) >= DECODE_HISTORY_MAX_BATCH) break;
if (startedAt > 0 && (performance.now() - startedAt) >= DECODE_HISTORY_SLICE_BUDGET_MS) break;
}
if (typeof onProgress === "function") {
onProgress(nextIndex, buffer.length);
}
if (nextIndex < buffer.length) {
scheduleDecodeHistoryDrainStep(() => drainDecodeHistory(buffer, nextIndex, onDone, onProgress));
} else if (typeof onDone === "function") {
onDone();
}
}
function loadDecodeHistoryOnMainThread(onReady, onError) { function loadDecodeHistoryOnMainThread(onReady, onError) {
fetch("/decode/history").then(async (resp) => { fetch("/decode/history").then(async (resp) => {
if (!resp.ok) return null; if (!resp.ok) return null;
setDecodeHistoryOverlayVisible(true, "Loading decode history…", "Receiving compressed history payload"); setDecodeHistoryOverlayVisible(true, "Loading decode history…", "Receiving compressed history payload");
const payload = await resp.arrayBuffer(); const payload = await resp.arrayBuffer();
if (!payload || payload.byteLength === 0) return []; if (!payload || payload.byteLength === 0) return {};
setDecodeHistoryOverlayVisible(true, "Loading decode history…", "Decoding compressed history payload"); setDecodeHistoryOverlayVisible(true, "Loading decode history…", "Decoding compressed history payload");
return decodeCborPayload(payload); return decodeCborPayload(payload);
}).then((msgs) => { }).then((groups) => {
if (typeof onReady === "function") onReady(Array.isArray(msgs) ? msgs : []); if (typeof onReady === "function") onReady(groups && typeof groups === "object" ? groups : {});
}).catch((err) => { }).catch((err) => {
if (typeof onError === "function") onError(err); if (typeof onError === "function") onError(err);
}); });
} }
function restoreDecodeHistoryGroup(kind, messages) {
if (!Array.isArray(messages) || messages.length === 0) return;
if (kind === "ais" && window.restoreAisHistory) {
window.restoreAisHistory(messages);
return;
}
if (kind === "vdes" && window.restoreVdesHistory) {
window.restoreVdesHistory(messages);
return;
}
if (kind === "aprs" && window.restoreAprsHistory) {
window.restoreAprsHistory(messages);
return;
}
if (kind === "hf_aprs" && window.restoreHfAprsHistory) {
window.restoreHfAprsHistory(messages);
return;
}
if (kind === "cw" && window.restoreCwHistory) {
window.restoreCwHistory(messages);
return;
}
if (kind === "ft8" && window.restoreFt8History) {
window.restoreFt8History(messages);
return;
}
if (kind === "wspr" && window.restoreWsprHistory) {
window.restoreWsprHistory(messages);
return;
}
}
function connectDecode() { function connectDecode() {
if (decodeSource) { decodeSource.close(); } if (decodeSource) { decodeSource.close(); }
terminateDecodeHistoryWorker(); terminateDecodeHistoryWorker();
@@ -7447,7 +7447,7 @@ function connectDecode() {
let historyBatchDrainScheduled = false; let historyBatchDrainScheduled = false;
let historyTotal = 0; let historyTotal = 0;
let historyProcessed = 0; let historyProcessed = 0;
const historyBatchQueue = []; const historyGroupQueue = [];
const liveBuffer = []; const liveBuffer = [];
function flushLiveBuffer() { function flushLiveBuffer() {
historySettled = true; historySettled = true;
@@ -7470,62 +7470,74 @@ function connectDecode() {
function maybeFinishHistoryReplay() { function maybeFinishHistoryReplay() {
if (historySettled) return; if (historySettled) return;
if (historyWorkerDone && historyBatchQueue.length === 0) { if (historyWorkerDone && historyGroupQueue.length === 0) {
clearTimeout(historyTimeout); clearTimeout(historyTimeout);
flushLiveBuffer(); flushLiveBuffer();
} }
} }
function pumpDecodeHistoryBatchQueue() { function pumpDecodeHistoryGroupQueue() {
historyBatchDrainScheduled = false; historyBatchDrainScheduled = false;
const startedAt = typeof performance !== "undefined" && typeof performance.now === "function" const startedAt = typeof performance !== "undefined" && typeof performance.now === "function"
? performance.now() ? performance.now()
: 0; : 0;
while (historyBatchQueue.length > 0) { while (historyGroupQueue.length > 0) {
const batch = historyBatchQueue.shift(); const next = historyGroupQueue.shift();
dispatchDecodeBatch(batch); restoreDecodeHistoryGroup(next.kind, next.messages);
historyProcessed += Array.isArray(batch) ? batch.length : 0; historyProcessed += Array.isArray(next.messages) ? next.messages.length : 0;
updateHistoryReplayOverlay(); updateHistoryReplayOverlay();
if (startedAt > 0 && (performance.now() - startedAt) >= DECODE_HISTORY_BATCH_DRAIN_BUDGET_MS) { if (startedAt > 0 && (performance.now() - startedAt) >= DECODE_HISTORY_BATCH_DRAIN_BUDGET_MS) {
break; break;
} }
} }
if (historyBatchQueue.length > 0) { if (historyGroupQueue.length > 0) {
scheduleDecodeHistoryDrainStep(pumpDecodeHistoryBatchQueue); scheduleDecodeHistoryDrainStep(pumpDecodeHistoryGroupQueue);
historyBatchDrainScheduled = true; historyBatchDrainScheduled = true;
return; return;
} }
maybeFinishHistoryReplay(); maybeFinishHistoryReplay();
} }
function enqueueDecodeHistoryBatch(batch) { function enqueueDecodeHistoryGroup(kind, messages) {
if (!Array.isArray(batch) || batch.length === 0) return; if (!Array.isArray(messages) || messages.length === 0) return;
historyBatchQueue.push(batch); historyGroupQueue.push({ kind, messages });
if (historyBatchDrainScheduled) return; if (historyBatchDrainScheduled) return;
historyBatchDrainScheduled = true; historyBatchDrainScheduled = true;
scheduleDecodeHistoryDrainStep(pumpDecodeHistoryBatchQueue); scheduleDecodeHistoryDrainStep(pumpDecodeHistoryGroupQueue);
}
function totalDecodeHistoryMessages(groups) {
if (!groups || typeof groups !== "object") return 0;
return ["ais", "vdes", "aprs", "hf_aprs", "cw", "ft8", "wspr"]
.reduce((sum, key) => sum + (Array.isArray(groups[key]) ? groups[key].length : 0), 0);
}
function enqueueDecodeHistoryGroups(groups) {
historyTotal = totalDecodeHistoryMessages(groups);
historyProcessed = 0;
if (historyTotal > 0) {
setDecodeHistoryReplayActive(true);
updateHistoryReplayOverlay();
}
for (const kind of ["ais", "vdes", "aprs", "hf_aprs", "cw", "ft8", "wspr"]) {
const messages = groups && Array.isArray(groups[kind]) ? groups[kind] : [];
if (messages.length === 0) continue;
for (let index = 0; index < messages.length; index += DECODE_HISTORY_WORKER_GROUP_LIMIT) {
enqueueDecodeHistoryGroup(kind, messages.slice(index, index + DECODE_HISTORY_WORKER_GROUP_LIMIT));
}
}
historyWorkerDone = true;
maybeFinishHistoryReplay();
} }
function startDecodeHistoryFallback() { function startDecodeHistoryFallback() {
if (historyFallbackStarted || historySettled) return; if (historyFallbackStarted || historySettled) return;
historyFallbackStarted = true; historyFallbackStarted = true;
loadDecodeHistoryOnMainThread((msgs) => { loadDecodeHistoryOnMainThread((groups) => {
clearTimeout(historyTimeout); clearTimeout(historyTimeout);
if (Array.isArray(msgs) && msgs.length > 0) { const total = totalDecodeHistoryMessages(groups);
setDecodeHistoryReplayActive(true); if (total > 0) {
setDecodeHistoryOverlayVisible(true, "Loading decode history…", `Replaying 0 / ${msgs.length} decoded messages`); enqueueDecodeHistoryGroups(groups);
drainDecodeHistory(
msgs,
0,
flushLiveBuffer,
(processed, total) => {
setDecodeHistoryOverlayVisible(
true,
"Loading decode history…",
`Replaying ${processed} / ${total} decoded messages`
);
}
);
} else { } else {
flushLiveBuffer(); flushLiveBuffer();
} }
@@ -7567,8 +7579,8 @@ function connectDecode() {
} }
return; return;
} }
if (data.type === "batch") { if (data.type === "group") {
enqueueDecodeHistoryBatch(data.batch); enqueueDecodeHistoryGroup(String(data.kind || ""), data.messages);
return; return;
} }
if (data.type === "done") { if (data.type === "done") {
@@ -7587,7 +7599,7 @@ function connectDecode() {
worker.postMessage({ worker.postMessage({
type: "fetch-history", type: "fetch-history",
url: "/decode/history", url: "/decode/history",
batchLimit: DECODE_HISTORY_TYPE_BATCH_LIMIT, batchLimit: DECODE_HISTORY_WORKER_GROUP_LIMIT,
}); });
return true; return true;
} }
src/trx-client/trx-frontend/trx-frontend-http/assets/web/decode-history-worker.js
+25 -38
View File
@@ -1,4 +1,5 @@
const textDecoder = typeof TextDecoder === "function" ? new TextDecoder() : null; const textDecoder = typeof TextDecoder === "function" ? new TextDecoder() : null;
const HISTORY_GROUP_KEYS = ["ais", "vdes", "aprs", "hf_aprs", "cw", "ft8", "wspr"];
function decodeCborUint(view, bytes, state, additional) { function decodeCborUint(view, bytes, state, additional) {
const offset = state.offset; const offset = state.offset;
@@ -111,13 +112,15 @@ function decodeCborItem(view, bytes, state) {
throw new Error("Unsupported CBOR major type"); throw new Error("Unsupported CBOR major type");
} }
function decodeTopLevelArrayLength(view, bytes, state) { function decodeCborPayload(buffer) {
if (state.offset >= bytes.length) throw new Error("CBOR payload truncated"); const bytes = buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer);
const initial = bytes[state.offset++]; const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
const major = initial >> 5; const state = { offset: 0 };
const additional = initial & 0x1f; const value = decodeCborItem(view, bytes, state);
if (major !== 4) throw new Error("Decode history payload is not a CBOR array"); if (state.offset !== bytes.length) {
return decodeCborUint(view, bytes, state, additional); throw new Error("Unexpected trailing bytes in decode history payload");
}
return value;
} }
async function fetchAndDecodeHistory(url, batchLimit) { async function fetchAndDecodeHistory(url, batchLimit) {
@@ -132,46 +135,30 @@ async function fetchAndDecodeHistory(url, batchLimit) {
} }
self.postMessage({ type: "status", phase: "decoding" }); self.postMessage({ type: "status", phase: "decoding" });
const bytes = new Uint8Array(payload); const history = decodeCborPayload(payload);
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); const total = HISTORY_GROUP_KEYS.reduce((sum, key) => {
const state = { offset: 0 }; const items = history && Array.isArray(history[key]) ? history[key] : [];
const total = decodeTopLevelArrayLength(view, bytes, state); return sum + items.length;
}, 0);
self.postMessage({ type: "start", total }); self.postMessage({ type: "start", total });
let processed = 0; let processed = 0;
let currentType = ""; const safeLimit = Math.max(1, Math.min(2048, Number(batchLimit) || 512));
let currentBatch = [];
const safeLimit = Math.max(1, Math.min(512, Number(batchLimit) || 192));
function flushBatch() { for (const kind of HISTORY_GROUP_KEYS) {
if (currentBatch.length === 0) return; const items = history && Array.isArray(history[kind]) ? history[kind] : [];
if (items.length === 0) continue;
for (let index = 0; index < items.length; index += safeLimit) {
const messages = items.slice(index, index + safeLimit);
processed += messages.length;
self.postMessage({ self.postMessage({
type: "batch", type: "group",
batch: currentBatch, kind,
messages,
processed, processed,
total, total,
}); });
currentBatch = [];
currentType = "";
} }
for (let i = 0; i < total; i += 1) {
const item = decodeCborItem(view, bytes, state);
const itemType = String(item?.type || "");
if (
currentBatch.length > 0
&& (itemType !== currentType || currentBatch.length >= safeLimit)
) {
flushBatch();
}
currentType = itemType;
currentBatch.push(item);
processed += 1;
}
flushBatch();
if (state.offset !== bytes.length) {
throw new Error("Unexpected trailing bytes in decode history payload");
} }
self.postMessage({ type: "done", total }); self.postMessage({ type: "done", total });
} }
src/trx-client/trx-frontend/trx-frontend-http/assets/web/plugins/ais.js
+4
View File
@@ -390,6 +390,10 @@ window.onServerAisBatch = function(messages) {
scheduleAisHistoryRender(); scheduleAisHistoryRender();
}; };
window.restoreAisHistory = function(messages) {
window.onServerAisBatch(messages);
};
window.pruneAisHistoryView = function() { window.pruneAisHistoryView = function() {
pruneAisMessageHistory(); pruneAisMessageHistory();
updateAisBar(); updateAisBar();
src/trx-client/trx-frontend/trx-frontend-http/assets/web/plugins/aprs.js
+4
View File
@@ -447,6 +447,10 @@ window.onServerAprsBatch = function(packets) {
scheduleAprsHistoryRender(); scheduleAprsHistoryRender();
}; };
window.restoreAprsHistory = function(packets) {
window.onServerAprsBatch(packets);
};
document.getElementById("aprs-clear-btn").addEventListener("click", async () => { document.getElementById("aprs-clear-btn").addEventListener("click", async () => {
try { try {
await postPath("/clear_aprs_decode"); await postPath("/clear_aprs_decode");
src/trx-client/trx-frontend/trx-frontend-http/assets/web/plugins/cw.js
+8
View File
@@ -426,6 +426,14 @@ window.onServerCw = function(evt) {
}); });
}; };
window.restoreCwHistory = function(events) {
if (!Array.isArray(events) || events.length === 0) return;
if (cwStatusEl) cwStatusEl.textContent = cwPaused ? "Paused" : "Receiving";
for (const evt of events) {
window.onServerCw(evt);
}
};
if (cwPauseBtn) { if (cwPauseBtn) {
cwPauseBtn.addEventListener("click", () => { cwPauseBtn.addEventListener("click", () => {
cwPaused = !cwPaused; cwPaused = !cwPaused;
src/trx-client/trx-frontend/trx-frontend-http/assets/web/plugins/ft8.js
+4
View File
@@ -167,6 +167,10 @@ window.onServerFt8Batch = function(messages) {
scheduleFt8HistoryRender(); scheduleFt8HistoryRender();
}; };
window.restoreFt8History = function(messages) {
window.onServerFt8Batch(messages);
};
window.pruneFt8HistoryView = function() { window.pruneFt8HistoryView = function() {
pruneFt8MessageHistory(); pruneFt8MessageHistory();
updateFt8Bar(); updateFt8Bar();
src/trx-client/trx-frontend/trx-frontend-http/assets/web/plugins/hf-aprs.js
+4
View File
@@ -393,6 +393,10 @@ window.onServerHfAprsBatch = function(packets) {
scheduleHfAprsHistoryRender(); scheduleHfAprsHistoryRender();
}; };
window.restoreHfAprsHistory = function(packets) {
window.onServerHfAprsBatch(packets);
};
document.getElementById("hf-aprs-decode-toggle-btn")?.addEventListener("click", async () => { document.getElementById("hf-aprs-decode-toggle-btn")?.addEventListener("click", async () => {
try { await postPath("/toggle_hf_aprs_decode"); } catch (e) { console.error("HF APRS toggle failed", e); } try { await postPath("/toggle_hf_aprs_decode"); } catch (e) { console.error("HF APRS toggle failed", e); }
}); });
src/trx-client/trx-frontend/trx-frontend-http/assets/web/plugins/vdes.js
+4
View File
@@ -331,6 +331,10 @@ window.onServerVdesBatch = function(messages) {
scheduleVdesHistoryRender(); scheduleVdesHistoryRender();
}; };
window.restoreVdesHistory = function(messages) {
window.onServerVdesBatch(messages);
};
if (vdesClearBtn) { if (vdesClearBtn) {
vdesClearBtn.addEventListener("click", async () => { vdesClearBtn.addEventListener("click", async () => {
try { try {
src/trx-client/trx-frontend/trx-frontend-http/assets/web/plugins/wspr.js
+4
View File
@@ -144,6 +144,10 @@ window.onServerWsprBatch = function(messages) {
scheduleWsprHistoryRender(); scheduleWsprHistoryRender();
}; };
window.restoreWsprHistory = function(messages) {
window.onServerWsprBatch(messages);
};
window.pruneWsprHistoryView = function() { window.pruneWsprHistoryView = function() {
pruneWsprMessageHistory(); pruneWsprMessageHistory();
renderWsprHistory(); renderWsprHistory();
src/trx-client/trx-frontend/trx-frontend-http/src/api.rs
+35 -28
View File
@@ -442,33 +442,40 @@ fn sync_scheduler_vchannels(
vchan_mgr.sync_scheduler_channels(rig_id, &desired); vchan_mgr.sync_scheduler_channels(rig_id, &desired);
} }
/// Build the combined decode history vector from all per-decoder ring-buffers. #[derive(serde::Serialize)]
fn collect_decode_history(context: &FrontendRuntimeContext) -> Vec<trx_core::decode::DecodedMessage> { struct DecodeHistoryPayload {
let ais = crate::server::audio::snapshot_ais_history(context); ais: Vec<trx_core::decode::AisMessage>,
let vdes = crate::server::audio::snapshot_vdes_history(context); vdes: Vec<trx_core::decode::VdesMessage>,
let aprs = crate::server::audio::snapshot_aprs_history(context); aprs: Vec<trx_core::decode::AprsPacket>,
let hf_aprs = crate::server::audio::snapshot_hf_aprs_history(context); hf_aprs: Vec<trx_core::decode::AprsPacket>,
let cw = crate::server::audio::snapshot_cw_history(context); cw: Vec<trx_core::decode::CwEvent>,
let ft8 = crate::server::audio::snapshot_ft8_history(context); ft8: Vec<trx_core::decode::Ft8Message>,
let wspr = crate::server::audio::snapshot_wspr_history(context); wspr: Vec<trx_core::decode::WsprMessage>,
}
let mut out = Vec::with_capacity( impl DecodeHistoryPayload {
ais.len() fn total_messages(&self) -> usize {
+ vdes.len() self.ais.len()
+ aprs.len() + self.vdes.len()
+ hf_aprs.len() + self.aprs.len()
+ cw.len() + self.hf_aprs.len()
+ ft8.len() + self.cw.len()
+ wspr.len(), + self.ft8.len()
); + self.wspr.len()
out.extend(ais.into_iter().map(trx_core::decode::DecodedMessage::Ais)); }
out.extend(vdes.into_iter().map(trx_core::decode::DecodedMessage::Vdes)); }
out.extend(aprs.into_iter().map(trx_core::decode::DecodedMessage::Aprs));
out.extend(hf_aprs.into_iter().map(trx_core::decode::DecodedMessage::HfAprs)); /// Build the grouped decode history payload from all per-decoder ring-buffers.
out.extend(cw.into_iter().map(trx_core::decode::DecodedMessage::Cw)); fn collect_decode_history(context: &FrontendRuntimeContext) -> DecodeHistoryPayload {
out.extend(ft8.into_iter().map(trx_core::decode::DecodedMessage::Ft8)); DecodeHistoryPayload {
out.extend(wspr.into_iter().map(trx_core::decode::DecodedMessage::Wspr)); ais: crate::server::audio::snapshot_ais_history(context),
out vdes: crate::server::audio::snapshot_vdes_history(context),
aprs: crate::server::audio::snapshot_aprs_history(context),
hf_aprs: crate::server::audio::snapshot_hf_aprs_history(context),
cw: crate::server::audio::snapshot_cw_history(context),
ft8: crate::server::audio::snapshot_ft8_history(context),
wspr: crate::server::audio::snapshot_wspr_history(context),
}
} }
fn encode_cbor_length(out: &mut Vec<u8>, major: u8, value: u64) { fn encode_cbor_length(out: &mut Vec<u8>, major: u8, value: u64) {
@@ -537,10 +544,10 @@ fn encode_cbor_json_value(out: &mut Vec<u8>, value: &serde_json::Value) {
} }
fn encode_decode_history_cbor( fn encode_decode_history_cbor(
history: &[trx_core::decode::DecodedMessage], history: &DecodeHistoryPayload,
) -> Result<Vec<u8>, serde_json::Error> { ) -> Result<Vec<u8>, serde_json::Error> {
let value = serde_json::to_value(history)?; let value = serde_json::to_value(history)?;
let mut out = Vec::with_capacity(history.len().saturating_mul(96)); let mut out = Vec::with_capacity(history.total_messages().saturating_mul(96));
encode_cbor_json_value(&mut out, &value); encode_cbor_json_value(&mut out, &value);
Ok(out) Ok(out)
} }