aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMunyoki Kilyungi2023-08-24 12:45:53 +0300
committerMunyoki Kilyungi2023-08-24 12:49:39 +0300
commita50c7d3479f8220301db4dcd7b7f07a1483d0020 (patch)
tree672bcc1ea7789d42713719ff12f976a097ce5764
parent8ccc7ba28046ed43ac7b8b5831b0eebb5a9b78c9 (diff)
downloadgn-transform-databases-a50c7d3479f8220301db4dcd7b7f07a1483d0020.tar.gz
Add new methods for bulk-loading data
Signed-off-by: Munyoki Kilyungi <me@bonfacemunyoki.com>
-rwxr-xr-xload-rdf.scm33
1 files changed, 31 insertions, 2 deletions
diff --git a/load-rdf.scm b/load-rdf.scm
index 6f469b8..5dcfd62 100755
--- a/load-rdf.scm
+++ b/load-rdf.scm
@@ -52,6 +52,35 @@ DELETE FROM rdf_quad WHERE g = iri_to_id ('~a');"
OPEN_WRITE
"isql"))
+(define (empty-load-queue port password)
+ "Empty the "
+ (call-with-pipe
+ (lambda (out)
+ (format out
+ "SET DSN=localhost:~a;
+SET PWD=~s;
+DELETE FROM DB.DBA.load_list;"
+ port
+ password))
+ OPEN_WRITE
+ "isql"))
+
+(define (bulk-load-data port password graph)
+ "Bulk load data into virtuoso"
+ (call-with-pipe
+ (lambda (out)
+ (format out
+ "SET DSN=localhost:~a;
+SET PWD=~s;
+ld_dir('/var/lib/data', '*.ttl', '~a');
+rdf_loader_run();
+CHECKPOINT;"
+ port
+ password
+ graph))
+ OPEN_WRITE
+ "isql"))
+
(define (time-thunk thunk)
"Run THUNK and return the time taken in seconds."
(let ((start-time (current-time)))
@@ -76,14 +105,14 @@ DELETE FROM rdf_quad WHERE g = iri_to_id ('~a');"
%graph-uri)))
;; Delete the load queue
(format (current-output-port)
- "Existing virtuoso data deleted in ~a seconds~%"
+ "Existing DB.LOAD queue deleted in ~a seconds~%"
(time-thunk
(cut empty-load-queue
(assq-ref connection-settings 'virtuoso-port)
(assq-ref connection-settings 'virtuoso-password))))
;; Bulk load data
(format (current-output-port)
- "Existing virtuoso data deleted in ~a seconds~%"
+ "Existing virtuoso data uploaded in ~a seconds~%"
(time-thunk
(cut bulk-load-data
(assq-ref connection-settings 'virtuoso-port)