From ba347362d64f9e634fad7f1aa8ab0a7753ff7a81 Mon Sep 17 00:00:00 2001 From: Munyoki Kilyungi Date: Mon, 2 Mar 2026 18:19:27 +0300 Subject: transform: add keyset pagination and streaming emit path * transform/special-forms.scm (define-transformer): Add keyset arguments and cursor return path; add SQL helpers for keyset ordering/filtering; avoid po-alist allocation by streaming short turtle emission. (with-documentation): Add keyset pagination loop using per-input cursor state and configurable chunk key field. (emit-short-turtle): Emit triples in a single linear pass. * examples/probesets-experiment-metadata.scm: Enable keyset pagination and set composite chunk key fields for ProbeSetFreeze/ProbeSet join. Signed-off-by: Munyoki Kilyungi --- examples/probesets-experiment-metadata.scm | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'examples') diff --git a/examples/probesets-experiment-metadata.scm b/examples/probesets-experiment-metadata.scm index b2eb93e..d7c0764 100755 --- a/examples/probesets-experiment-metadata.scm +++ b/examples/probesets-experiment-metadata.scm @@ -70,10 +70,11 @@ (name "ProbeSet Experiments Metadata") (connection %connection-settings) (table-metadata? #f) - (total-rows (assoc-ref - (sql-find db "SELECT count(*) AS count from ProbeSetXRef") - "count")) - (rows-per-chunk 1000000) + ;; Use composite keyset pagination over joined table identifiers. + (keyset-pagination? #t) + (chunk-key-field '("ProbeSetFreeze.Id" "ProbeSet.Id")) + ;; (total-rows 1000000) + (rows-per-chunk 10000) (prefixes '(("dcat:" "") ("gn:" "") -- cgit 1.4.1