plc-replica/scrape.ts

38 lines
958 B
TypeScript
Raw Normal View History

2024-11-27 16:30:50 +00:00
import { TextLineStream } from "jsr:@std/streams@1/text-line-stream";
import { ExportEntry } from "./directory-tailer.ts";
import { tailer } from "./main.ts";
export const catchUp = async () => {
using exports = await Deno.open("./data/exports.jsonl", { read: true });
await exports.seek(-4096, Deno.SeekMode.End); // grab last 4kib
const lineStream = exports.readable
.pipeThrough(new TextDecoderStream())
.pipeThrough(new TextLineStream());
// discard one partial line
{
const lineReader = lineStream.getReader();
const _line = await lineReader.read();
lineReader.releaseLock();
}
let lastLine: string | undefined;
for await (const line of lineStream.values()) {
lastLine = line;
}
if (lastLine) {
const entry = JSON.parse(lastLine) as unknown as ExportEntry;
tailer.latestDate = entry.createdAt;
}
};
try {
await catchUp();
} catch (err) {
console.warn(err);
}
await tailer.fetchExports();