1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
|
#! /usr/bin/env guile
!#
(use-modules (srfi srfi-1)
(srfi srfi-26)
(ice-9 getopt-long)
(ice-9 match)
(ice-9 regex)
(transform strings)
(transform sql)
(transform triples)
(transform special-forms))
(define (remap-species-identifiers str)
"This procedure remaps identifiers to standard binominal. Obviously this should
be sorted by correcting the database!"
(match str
["Fly (Drosophila melanogaster dm6)" "Drosophila melanogaster"]
["Oryzias latipes (Japanese medaka)" "Oryzias latipes"]
["Macaca mulatta" "Macaca nemestrina"]
["Bat (Glossophaga soricina)" "Glossophaga soricina"]
[str str]))
;; One email ID in the Investigators table has spaces in it. This
;; function fixes that.
(define (fix-email-id email)
(string-delete #\space email))
(define (investigator-attributes->id first-name last-name email)
;; There is just one record corresponding to "Evan Williams" which
;; does not have an email ID. To accommodate that record, we
;; construct the investigator ID from not just the email ID, but
;; also the first and the last names. It would be preferable to just
;; find Evan Williams' email ID and insert it into the database.
(string->identifier "investigator"
(string-join
(list first-name last-name (fix-email-id email))
"_")))
(define-transformer investigators
;; There are a few duplicate entries. We group by email to
;; deduplicate.
(tables (Investigators)
"GROUP BY Email")
(triples (investigator-attributes->id (field Investigators FirstName)
(field Investigators LastName)
(field Investigators Email))
(set rdf:type 'foaf:Person)
(set foaf:name (string-append (field Investigators FirstName) " "
(field Investigators LastName)))
(set foaf:givenName
(field Investigators FirstName))
(set foaf:familyName
(field Investigators LastName))
(set foaf:homepage (field Investigators Url))
(set v:adr (field Investigators Address))
(set v:locality (field Investigators City))
(set v:region (field Investigators State))
(set v:postal-code (field Investigators ZipCode))
(set v:country-name (field Investigators Country))))
(define-transformer gene-chip
(tables (GeneChip
(left-join Species "USING (SpeciesId)")))
(schema-triples
(gnc:geneChip a skos:Concept)
(gnc:geneChip
skos:description
"This is a set of controlled terms that are used to describe a given gene chip/platform")
(gnt:hasGeoSeriesId rdfs:domain gnc:platform)
(gnt:hasGeoSeriesId rdfs:domain gnc:geneChip)
(gnt:hasGOTreeValue a owl:ObjectProperty)
(gnt:hasGOTreeValue skos:definition "This resource the following GO tree value")
(gnt:hasGOTreeValue rdfs:domain gnc:geneChip))
(triples (string->identifier "platform" (field GeneChip Name))
(set rdf:type 'gnc:geneChip)
(set rdfs:label (field GeneChip GeneChipName))
(set skos:prefLabel (field GeneChip Name))
(set skos:altLabel (field ("IF(GeneChip.GeneChipName != GeneChip.Title, Title, NULL)"
Title)))
(set gnt:hasGOTreeValue (field GeneChip Go_tree_value))
(set xkos:classifiedUnder
(string->identifier "" (remap-species-identifiers (field Species Fullname))
#:separator ""
#:proc string-capitalize-first))
(set gnt:hasGeoSeriesId
(ontology 'geoSeries:
(string-trim-both (field GeneChip GeoPlatform))))))
(define-transformer info-files
(tables (InfoFiles
(left-join PublishFreeze "ON InfoFiles.InfoPageName = PublishFreeze.Name")
(left-join GenoFreeze "ON InfoFiles.InfoPageName = GenoFreeze.Name")
(left-join ProbeSetFreeze "ON InfoFiles.InfoPageName = ProbeSetFreeze.Name")
(left-join InbredSet "ON InfoFiles.InbredSetId = InbredSet.InbredSetId")
(left-join Species "ON InfoFiles.SpeciesId = Species.SpeciesId")
(left-join Datasets "USING (DatasetId)")
(left-join DatasetStatus "USING (DatasetStatusId)")
(left-join Tissue "USING (TissueId)")
(left-join Investigators "USING (InvestigatorId)")
(left-join AvgMethod "USING (AvgMethodId)")
(left-join Organizations "USING (OrganizationId)")
(left-join GeneChip "USING (GeneChipId)"))
;; XXXX: There are datasets that don't have the InbredSetId
;; in the Infofiles table. This clause allows us to check
;; if they exist in the (Publish/Geno)Freeze tables.
"LEFT JOIN InbredSet PublishInbredSet ON PublishFreeze.InbredSetId = PublishInbredSet.InbredSetId LEFT JOIN InbredSet GenoInbredSet ON GenoFreeze.InbredSetId = GenoInbredSet.InbredSetId WHERE GN_AccesionId IS NOT NULL")
(schema-triples
(gnt:hasTissue rdfs:domain dcat:Dataset)
(gnt:hasTissue a owl:ObjectProperty)
(gnt:hasTissue skos:definition "Tissues this resource has")
(gnt:usesNormalization rdfs:domain dcat:Dataset)
(gnt:usesNormalization a owl:ObjectProperty)
(gnt:usesNormalization skos:definition "Normalization techniques this resource has")
(gnt:usesPlatform rdfs:domain dcat:Dataset)
(gnt:usesPlatform a owl:ObjectProperty)
(gnt:usesPlatform skos:definition "The Platform this resource uses")
(gnt:hasGeoSeriesId rdfs:domain dcat:Dataset)
(gnt:hasGeoSeriesId a owl:ObjectProperty)
(gnt:hasGeoSeriesId skos:definition "id of record in NCBI database")
(gnt:hasExperimentType rdfs:domain dcat:Dataset)
(gnt:hasExperimentType a owl:ObjectProperty)
(gnt:hasExperimentType rdfs:label "Experiment Type Metadata")
(gnt:hasExperimentType skos:definition "Information about the experiment type")
(gnt:hasTissueInfo rdfs:domain dcat:Dataset)
(gnt:hasTissueInfo a owl:ObjectProperty)
(gnt:hasTissueInfo skos:definition "Metadata about Tissue for this resource")
(gnt:hasExperimentDesignInfo rdfs:domain dcat:Dataset)
(gnt:hasExperimentDesignInfo rdfs:label "Experiment Design")
(gnt:hasExperimentDesignInfo a owl:ObjectProperty)
(gnt:hasExperimentDesignInfo skos:definition "Information about how the experiment was designed")
(gnt:hasNotes rdfs:domain dcat:Dataset)
(gnt:hasNotes a owl:ObjectProperty)
(gnt:hasNotes rdfs:label "Notes")
(gnt:hasNotes skos:definition "Extra Notes about this dataset")
(gnt:hasDataProcessingInfo rdfs:domain dcat:Dataset)
(gnt:hasDataProcessingInfo rdfs:label "About Data Processing")
(gnt:hasDataProcessingInfo a owl:ObjectProperty)
(gnt:hasDataProcessingInfo skos:definition "Information about how this dataset was processed")
(gnt:hasPlatformInfo rdfs:domain dcat:Dataset)
(gnt:hasPlatformInfo a owl:ObjectProperty)
(gnt:hasPlatformInfo rdfs:label "About Platform")
(gnt:hasPlatformInfo skos:definition "Information about the platform that was used with this dataset")
(gnt:hasCaseInfo rdfs:domain dcat:Dataset)
(gnt:hasCaseInfo rdfs:label "About Case")
(gnt:hasCaseInfo a owl:ObjectProperty)
(gnt:hasCaseInfo skos:definition "Information about the cases used in this platform")
(gnt:hasExperimentType skos:definition "Information about the experiment type")
(gnt:hasAcknowledgement rdfs:domain dcat:Dataset)
(gnt:hasAcknowledgement rdfs:label "Acknowledgement")
(gnt:hasAcknowledgement a owl:ObjectProperty)
(gnt:hasAcknowledgement skos:definition "People to acknowledge"))
(triples (string->identifier
"" (regexp-substitute/global #f "[^A-Za-z0-9:]"
(field InfoFiles InfoPageName)
'pre "_" 'post))
(set rdf:type 'dcat:Dataset)
(set xkos:classifiedUnder
(let ([dataset-type
(string-trim-both
(field ("IF(GenoFreeze.Id IS NOT NULL, 'gnc:Genotype', IF(PublishFreeze.Id IS NOT NULL, 'gnc:Phenotype', IF(ProbeSetFreeze.Name IS NOT NULL, 'gnc:Probeset', '')))"
DatasetType)))])
(if (not (string-null? dataset-type))
(string->symbol
dataset-type)
"")))
(set rdfs:label (regexp-substitute/global
#f "^[Nn]one$"
(field InfoFiles InfoPageName)
""))
(set skos:prefLabel
(field ("IFNULL(GenoFreeze.FullName, IFNULL(PublishFreeze.FullName, ''))"
DatasetFullName)))
(set skos:altLabel (field Datasets DatasetName DatasetGroup))
(set dct:title
(regexp-substitute/global
#f "^[Nn]one$"
(or
(regexp-substitute/global
#f "^Unpublished$" (field Datasets PublicationTitle) "")
(field InfoFiles InfoFileTitle)
"")
""))
(set dct:created
(field ("IFNULL(GenoFreeze.CreateTime, IFNULL(PublishFreeze.CreateTime, IFNULL(ProbeSetFreeze.CreateTime, '')))"
createTimeGenoFreeze)))
(set dcat:contactPoint
(investigator-attributes->id (field Investigators FirstName)
(field Investigators LastName)
(field Investigators Email)))
(set foaf:Organization
(field Organizations OrganizationName))
(set dct:identifier (format #f "GN~a" (field InfoFiles GN_AccesionId)))
(set dct:accessRights (string-downcase
(field DatasetStatus DatasetStatusName)))
(set gnt:belongsToGroup
(string->identifier
"set"
(field ("IFNULL(InbredSet.Name, IFNULL(PublishInbredSet.Name, GenoInbredSet.Name))"
InbredSetName))))
(set gnt:hasTissue (string->identifier "tissue"
(field Tissue Short_Name)))
(set gnt:usesNormalization
(string->identifier "avgMethod"
;; If AvgMethodName is NULL, assume N/A.
(if (string-blank? (field AvgMethod Name AvgMethodName))
"N/A" (field AvgMethod Name AvgMethodName))))
(set gnt:usesPlatform
(string->identifier "platform"
(field GeneChip Name GeneChip)))
(set gnt:hasGeoSeriesId
(let ((s
(string-match "GSE[0-9]*"
(field ("IFNULL(Datasets.GeoSeries, '')" GeoSeries)))))
(if s (ontology
'geoSeries: (match:substring s))
"")))))
;; These are phenotype datasets that don't have Infofile metadata
(define-transformer publishfreeze
(tables (PublishFreeze
(left-join InfoFiles "ON InfoFiles.InfoPageName = PublishFreeze.Name")
(left-join InbredSet "ON PublishFreeze.InbredSetId = InbredSet.InbredSetId"))
"WHERE PublishFreeze.public > 0 AND PublishFreeze.confidentiality < 1 AND InfoFiles.InfoFileId IS NULL")
(triples
(string->identifier
""
(regexp-substitute/global #f "[^A-Za-z0-9:]"
(field PublishFreeze Name)
'pre "_" 'post))
(set rdf:type 'dcat:Dataset)
(set xkos:classifiedUnder 'gnc:Phenotype)
(set dct:title (field PublishFreeze FullName))
(set rdfs:label (field PublishFreeze Name))
(set skos:altLabel (field PublishFreeze ShortName))
(set dct:created (annotate-field
(field PublishFreeze CreateTime)
'^^xsd:date))
(set gnt:belongsToGroup
(string->identifier
"set" (field InbredSet Name InbredSetName)
#:separator ""
#:proc string-capitalize-first))))
(define-transformer genofreeze
(tables (GenoFreeze
(left-join InfoFiles "ON InfoFiles.InfoPageName = GenoFreeze.Name")
(left-join InbredSet "ON GenoFreeze.InbredSetId = InbredSet.InbredSetId"))
"WHERE GenoFreeze.public > 0 AND GenoFreeze.confidentiality < 1 AND InfoFiles.InfoPageName IS NULL")
(triples
(string->identifier
""
(regexp-substitute/global
#f "[^A-Za-z0-9:]"
(regexp-substitute/global
#f "[^A-Za-z0-9:]"
(field GenoFreeze Name)
'pre "_" 'post)
'pre "_" 'post))
(set rdf:type 'dcat:Dataset)
(set xkos:classifiedUnder 'gnc:Genotype)
(set rdfs:label (field GenoFreeze Name))
(set dct:title (field GenoFreeze FullName))
(set skos:altLabel (field GenoFreeze ShortName))
(set dct:created (annotate-field
(field GenoFreeze CreateTime)
'^^xsd:date))
(set gnt:belongsToGroup
(string->identifier
"set" (field InbredSet Name InbredSetName)
#:separator ""
#:proc string-capitalize-first))))
;; Molecular Traits are also referred to as ProbeSets
(define-transformer probesetfreeze
(tables (ProbeSetFreeze
(left-join InfoFiles "ON InfoFiles.InfoPageName = ProbeSetFreeze.Name")
(left-join ProbeFreeze "USING (ProbeFreezeId)")
(left-join AvgMethod "ON AvgMethod.AvgMethodId = ProbeSetFreeze.AvgID")
(left-join InbredSet "ON ProbeFreeze.InbredSetId = InbredSet.Id")
(left-join Tissue "ON ProbeFreeze.TissueId = Tissue.TissueId"))
"WHERE ProbeSetFreeze.public > 0 AND InfoFiles.InfoPageName IS NULL GROUP BY ProbeFreeze.Id")
(schema-triples
(gnt:usesNormalization rdfs:domain gnc:probeset)
(gnt:usesDataScale rdfs:domain gnc:probeset)
(gnt:usesDataScale a owl:ObjectProperty)
(gnt:usesDataScale skos:definition "Thi data scale this resource uses"))
(triples
(string->identifier
""
(regexp-substitute/global
#f "[^A-Za-z0-9:]"
(field ProbeSetFreeze Name)
'pre "_" 'post))
(set rdf:type 'dcat:Dataset)
(set xkos:classifiedUnder 'gnc:Probeset)
(set gnt:usesNormalization
(string->identifier "avgMethod"
;; If AvgMethodName is NULL, assume N/A.
(if (string-blank? (field AvgMethod Name AvgMethodName))
"N/A" (field AvgMethod Name AvgMethodName))))
(set dct:title (field ProbeSetFreeze FullName))
(set rdfs:label (field ProbeSetFreeze ShortName))
(set skos:prefLabel (field ProbeSetFreeze Name))
(set skos:altLabel (field ProbeSetFreeze Name2))
(set dct:created (annotate-field
(field ProbeSetFreeze CreateTime)
'^^xsd:datetime))
(set gnt:usesDataScale (field ProbeSetFreeze DataScale))
(set gnt:hasTissue
(string->identifier
"tissue"
(field Tissue Short_Name)))
(set gnt:belongsToGroup
(string->identifier
"set" (field InbredSet Name InbredSetName)
#:separator ""
#:proc string-capitalize-first))))
(let* ((option-spec
'((settings (single-char #\s) (value #t))
(output (single-char #\o) (value #t))
(documentation (single-char #\d) (value #t))))
(options (getopt-long (command-line) option-spec))
(settings (option-ref options 'settings #f))
(output (option-ref options 'output #f))
(documentation (option-ref options 'documentation #f))
(%connection-settings
(call-with-input-file settings
read)))
(with-documentation
(name "Info files / Investigators Metadata")
(connection %connection-settings)
(table-metadata? #f)
(prefixes
'(("v:" "<http://www.w3.org/2006/vcard/ns#>")
("foaf:" "<http://xmlns.com/foaf/0.1/>")
("xsd:" "<http://www.w3.org/2001/XMLSchema#>")
("dcat:" "<http://www.w3.org/ns/dcat#>")
("skos:" "<http://www.w3.org/2004/02/skos/core#>")
("xkos:" "<http://rdf-vocabulary.ddialliance.org/xkos#>")
("geoSeries:" "<http://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=>")
("gnt:" "<http://genenetwork.org/term/>")
("gn:" "<http://genenetwork.org/id/>")
("gnc:" "<http://genenetwork.org/category/>")
("rdf:" "<http://www.w3.org/1999/02/22-rdf-syntax-ns#>")
("owl:" "<http://www.w3.org/2002/07/owl#>")
("rdfs:" "<http://www.w3.org/2000/01/rdf-schema#>")
("taxon:" "<http://purl.uniprot.org/taxonomy/>")
("dct:" "<http://purl.org/dc/terms/>")))
(inputs
(list info-files
publishfreeze
genofreeze
probesetfreeze
investigators
gene-chip))
(outputs
`(#:documentation ,documentation
#:rdf ,output))))
|
