bcquality/custom/knowledge/integration/use-a-framing-record-for-inbound-polling.good.al

// Best practice: one framing record per feed. It remembers where the last run stopped,
// caps how much one run pulls, and carries a lock with a stale timeout so two overlapping
// Job Queue runs can never fetch the same window.

table 50130 "Inbound Feed Frame"
{
    DataClassification = SystemMetadata;

    fields
    {
        field(1; "Feed Code"; Code[20]) { Caption = 'Feed Code'; }
        // The watermark: the next run starts here, so no run ever re-pulls old data.
        field(2; "Last Fetch At"; DateTime) { Caption = 'Last Fetch At'; }
        // The cap: a long-quiet feed catches up over several bounded runs instead of one huge pull.
        field(3; "Max Window (Hours)"; Integer) { Caption = 'Max Window (Hours)'; }
        // Opaque continuation token from the source's paged API, when it offers one.
        field(4; "Cursor"; Text[250]) { Caption = 'Cursor'; }
        // The lock: a second concurrent run sees this set and backs off.
        field(5; "Locked"; Boolean) { Caption = 'Locked'; }
        // Stamped when the lock is taken, so a crashed run's lock can be reclaimed after a timeout.
        field(6; "Locked At"; DateTime) { Caption = 'Locked At'; }
    }

    keys { key(PK; "Feed Code") { Clustered = true; } }
}

codeunit 50131 "Inbound Poller"
{
    TableNo = "Job Queue Entry";

    procedure Poll(FeedCode: Code[20])
    var
        Frame: Record "Inbound Feed Frame";
        WindowEnd: DateTime;
    begin
        Frame.Get(FeedCode);

        // Acquire the lock first. If another run already owns this feed, exit quietly:
        // overlap is the whole problem we are preventing.
        if not TryAcquireLock(Frame) then
            exit;

        // Bounded incremental window: from the watermark up to a capped end. Never "fetch all".
        WindowEnd := CapWindow(Frame."Last Fetch At", Frame."Max Window (Hours)");
        FetchAndStage(Frame, Frame."Last Fetch At", WindowEnd);

        // Advance the watermark and cursor so the NEXT run starts exactly where this one ended.
        // Sequential runs therefore never overlap their windows.
        Frame."Last Fetch At" := WindowEnd;
        Frame."Cursor" := NextCursor();
        ReleaseLock(Frame);
    end;

    local procedure TryAcquireLock(var Frame: Record "Inbound Feed Frame"): Boolean
    begin
        // If the lock is held AND fresh, someone is actively polling: do not steal it.
        if Frame."Locked" and (CurrentDateTime() - Frame."Locked At" < GetStaleTimeoutMs()) then
            exit(false);

        // Either free, or stale (a previous run crashed without releasing). Reclaim it.
        Frame."Locked" := true;
        Frame."Locked At" := CurrentDateTime();
        Frame.Modify(true);
        Commit(); // make the lock durable before the long fetch starts
        exit(true);
    end;

    local procedure ReleaseLock(var Frame: Record "Inbound Feed Frame")
    begin
        Frame."Locked" := false;
        Frame.Modify(true);
    end;

    local procedure GetStaleTimeoutMs(): Integer
    begin
        exit(300000); // 5 minutes: longer than a healthy run, short enough to recover from a crash
    end;
}