diff options
| -rwxr-xr-x | dump.scm | 702 | ||||
| -rwxr-xr-x | examples/dump-dataset-metadata.scm | 220 | ||||
| -rwxr-xr-x | examples/dump-generif.scm | 159 | ||||
| -rwxr-xr-x | examples/dump-phenotype.scm | 131 | ||||
| -rwxr-xr-x | examples/dump-probeset.scm | 213 | ||||
| -rwxr-xr-x | examples/dump-publication.scm | 90 | ||||
| -rwxr-xr-x | examples/dump-species-metadata.scm | 141 | ||||
| -rwxr-xr-x | examples/dump-tissue.scm | 66 |
8 files changed, 1020 insertions, 702 deletions
diff --git a/dump.scm b/dump.scm deleted file mode 100755 index 0b2318c..0000000 --- a/dump.scm +++ /dev/null @@ -1,702 +0,0 @@ -#! /usr/bin/env guile -!# - -(use-modules (rnrs programs) - (rnrs io ports) - (srfi srfi-1) - (srfi srfi-26) - (ice-9 match) - (ice-9 regex) - (dump strings) - (dump sql) - (dump triples) - (dump special-forms)) - - -;;; GeneNetwork database connection parameters and dump path - -(define %connection-settings - (call-with-input-file (list-ref (command-line) 1) - read)) - -(define %dump-directory - (list-ref (command-line) 2)) - - -;;; Dump tables - -(define binomial-name->species-id - (cut string->identifier "species" <>)) - -(define-dump dump-species - (tables (Species)) - (schema-triples - (gn:name rdfs:range rdfs:Literal) - (gn:displayName rdfs:range rdfs:Literal) - (gn:binomialName rdfs:range rdfs:Literal) - (gn:family rdfs:range rdfs:Literal)) - (triples (binomial-name->species-id (field Species FullName)) - (set rdf:type 'gn:species) - (set gn:name (field Species SpeciesName)) - (set gn:displayName (field Species MenuName)) - (set gn:binomialName (field Species FullName)) - (set gn:family (field Species Family)) - (set up:organism (ontology 'taxon: (field Species TaxonomyId))))) - -(define-dump dump-strain - (tables (Strain - (join Species "ON Strain.SpeciesId = Species.SpeciesId"))) - (schema-triples - (gn:strainOfSpecies rdfs:domain gn:strain) - (gn:strainOfSpecies rdfs:range gn:species) - (gn:name rdfs:range rdfs:Literal) - (gn:alias rdfs:range rdfs:Literal) - (gn:symbol rdfs:range rdfs:Literal)) - (triples (string->identifier "strain" (field Strain Name)) - (set rdf:type 'gn:strain) - (set gn:strainOfSpecies - (binomial-name->species-id (field Species FullName))) - ;; Name, and maybe a second name - (set gn:name (field Strain Name)) - (set gn:name (field Strain Name2)) - (set gn:alias (field Strain Alias)) - (set gn:symbol (field Strain Symbol)))) - -;; TODO: This function is unused. Remove if not required. -(define mapping-method-name->id - (cut string->identifier "mappingMethod" <>)) - -;; TODO: This function is unused. Remove if not required. -(define-dump dump-mapping-method - (tables (MappingMethod)) - (triples (string->identifier "mappingMethod" (field MappingMethod Name)) - (set rdf:type 'gn:mappingMethod))) - -(define inbred-set-name->id - (cut string->identifier "inbredSet" <>)) - -(define-dump dump-inbred-set - (tables (InbredSet - (left-join Species "ON InbredSet.SpeciesId=Species.Id") - (left-join MappingMethod - "ON InbredSet.MappingMethodId=MappingMethod.Id"))) - (schema-triples - (gn:fullName rdfs:range rdfs:Literal) - (gn:geneticType rdfs:range rdfs:Literal) - (gn:inbredSetCode rdfs:range rdfs:Literal) - (gn:inbredFamily rdfs:range rdfs:Literal) - (gn:inbredSetOfSpecies rdfs:range gn:species) - (gn:inbredSetType rdfs:range rdfs:Literal) - (gn:phenotype rdfs:range gn:inbredSetType) - (gn:genotype rdfs:range gn:inbredSetType) - (gn:inbredSetOfMappingMethod rdfs:range gn:mappingMethod)) - (triples (inbred-set-name->id (field InbredSet Name)) - (set rdf:type 'gn:inbredSet) - (set gn:fullName (field InbredSet FullName)) - (set gn:geneticType (field InbredSet GeneticType)) - (set gn:inbredFamily (field InbredSet Family)) - (set gn:inbredSetOfMappingMethod (field MappingMethod Name)) - (set gn:inbredSetCode (field InbredSet InbredSetCode)) - (set gn:inbredSetOfSpecies - (binomial-name->species-id (field Species FullName BinomialName))) - (set gn:genotype - (field ("IF ((SELECT PublishFreeze.Name FROM PublishFreeze WHERE PublishFreeze.InbredSetId = InbredSet.Id LIMIT 1) IS NOT NULL, 'Traits and Cofactors', '')" genotypeP))) - (set gn:phenotype - (field ("IF ((SELECT GenoFreeze.Name FROM GenoFreeze WHERE GenoFreeze.InbredSetId = InbredSet.Id LIMIT 1) IS NOT NULL, 'DNA Markers and SNPs', '')" phenotypeP))))) - -(define-dump dump-molecular-traits - (tables (ProbeFreeze - (left-join ProbeSetFreeze "USING (ProbeFreezeId)") - (left-join InbredSet "USING (InbredSetId)") - (left-join Tissue "USING (TissueId)") - (left-join Species "USING (SpeciesId)")) - "GROUP BY InbredSet.Name") - (schema-triples - (gn:molecularTrait rdfs:range rdfs:Literal)) - (triples (inbred-set-name->id (field InbredSet Name)) - (set gn:molecularTrait (field Tissue TissueName)))) - -;; Metadata for published datasets -(define-dump dump-publishfreeze - (tables (PublishFreeze - (left-join InbredSet "USING (InbredSetId)"))) - (schema-triples - (gn:datasetOfInbredSet rdfs:range gn:inbredSet) - (gn:name rdfs:range rdfs:Literal) - (gn:fullName rdfs:range rdfs:Literal) - (gn:shortName rdfs:range rdfs:Literal) - (gn:createTime rdfs:range rdfs:Literal)) - (triples (string->identifier "dataset" (field PublishFreeze Name)) - (set rdf:type 'gn:dataset) - (set gn:name (field PublishFreeze Name)) - (set gn:fullName (field PublishFreeze FullName)) - (set gn:shortName (field PublishFreeze ShortName)) - (set gn:createTime (field PublishFreeze CreateTime)) - (set gn:datasetOfInbredSet - (inbred-set-name->id (field InbredSet Name InbredSetName))))) - -(define-dump dump-publication - (tables (Publication)) - (schema-triples - (gn:pubMedId rdfs:range rdfs:Literal) - (gn:title rdfs:range rdfs:Literal) - (gn:journal rdfs:range rdfs:Literal) - (gn:volume rdfs:range rdfs:Literal) - (gn:pages rdfs:range rdfs:Literal) - (gn:month rdfs:range rdfs:Literal) - (gn:year rdfs:range rdfs:Literal) - (gn:author rdfs:range rdfs:Literal) - (gn:abstract rdfs:range rdfs:Literal)) - (triples - (let ((pmid (field - ("IF(Publication.PubMed_ID IS NULL, '', CONVERT(Publication.PubMed_Id, INT))" - pmid))) - (publication-id (field Publication Id))) - (if (string-null? pmid) - (string->identifier "publication" - (number->string publication-id)) - (ontology 'pubmed: pmid))) - (set rdf:type 'gn:publication) - (set gn:pubMedId (field ("IFNULL(PubMed_ID, '')" pubmedId))) - (set gn:title (field Publication Title)) - (set gn:journal (field Publication Journal)) - (set gn:volume (field Publication Volume)) - (set gn:pages (field Publication Pages)) - (set gn:month (field Publication Month)) - (set gn:year (field Publication Year)) - (multiset gn:author - ;; The authors field is a comma - ;; separated list. Split it. - (map string-trim (string-split (sanitize-rdf-string (field Publication Authors)) #\,))) - (set gn:abstract - (sanitize-rdf-string (field Publication Abstract))))) - -(define tissue-short-name->id - (cut string->identifier "tissue" <>)) - -(define-dump dump-tissue - ;; The Name and TissueName fields seem to be identical. BIRN_lex_ID - ;; and BIRN_lex_Name are mostly NULL. - (tables (Tissue)) - (schema-triples - (gn:name rdfs:range rdfs:Literal)) - ;; Hopefully the Short_Name field is distinct and can be used as an - ;; identifier. - (triples (tissue-short-name->id (field Tissue Short_Name)) - (set rdf:type 'gn:tissue) - (set gn:name (field Tissue Name)))) - -;; One email ID in the Investigators table has spaces in it. This -;; function fixes that. -(define (fix-email-id email) - (string-delete #\space email)) - -(define (investigator-attributes->id first-name last-name email) - ;; There is just one record corresponding to "Evan Williams" which - ;; does not have an email ID. To accommodate that record, we - ;; construct the investigator ID from not just the email ID, but - ;; also the first and the last names. It would be preferable to just - ;; find Evan Williams' email ID and insert it into the database. - (string->identifier "investigator" - (string-join - ;; Add special case for Yohan Bossé whose name - ;; has unprintable characters. - ;; TODO: Fix Yohan Bossé's name in the database. - (let ((last-name (if (string=? first-name "Yohan") - "Bosse" - last-name))) - (list first-name last-name (fix-email-id email))) - "_"))) - -(define-dump dump-investigators - ;; There are a few duplicate entries. We group by email to - ;; deduplicate. - (tables (Investigators) - "GROUP BY Email") - (schema-triples - ;; TODO: Are ranges required for FOAF predicates? Can they not be - ;; obtained from the FOAF spec? - (foaf:name rdfs:range rdfs:Literal) - (foaf:givenName rdfs:range rdfs:Literal) - (foaf:familyName rdfs:range rdfs:Literal) - (foaf:phone rdfs:range rdfs:Literal) - (foaf:mbox rdfs:range rdfs:Literal) - (foaf:homepage rdfs:range rdfs:Literal) - (gn:address rdfs:range rdfs:Literal) - (gn:city rdfs:range rdfs:Literal) - (gn:state rdfs:range rdfs:Literal) - (gn:zipCode rdfs:range rdfs:Literal) - (gn:country rdfs:range rdfs:Literal)) - (triples (investigator-attributes->id (field Investigators FirstName) - (field Investigators LastName) - (field Investigators Email)) - (set rdf:type 'foaf:Person) - ;; Special case Yohan Bossé's name since the last name has - ;; unprintable characters. - (set foaf:name (string-append (field Investigators FirstName) " " - (if (string=? (field Investigators FirstName) "Yohan") - "Bossé" - (field Investigators LastName)))) - (set foaf:givenName (field Investigators FirstName)) - ;; Special case Yohan Bossé's name since the last name has - ;; unprintable characters. - (set foaf:familyName (if (string=? (field Investigators FirstName) "Yohan") - "Bossé" - (field Investigators LastName))) - (set foaf:phone (field Investigators Phone)) - (set foaf:mbox (fix-email-id (field Investigators Email))) - (set foaf:homepage (field Investigators Url)) - (set gn:address (field Investigators Address)) - (set gn:city (field Investigators City)) - (set gn:state (field Investigators State)) - (set gn:zipCode (field Investigators ZipCode)) - (set gn:country (field Investigators Country)))) - -(define avg-method-name->id - (cut string->identifier "avgmethod" <>)) - -(define-dump dump-avg-method - ;; The Name and Normalization fields seem to be the same. Dump only - ;; the Name field. - (tables (AvgMethod)) - (schema-triples - (gn:name rdfs:range rdfs:Literal)) - (triples (avg-method-name->id (field AvgMethod Name)) - (set rdf:type 'gn:avgMethod) - (set gn:name (field AvgMethod Name)))) - -(define gene-chip-name->id - (cut string->identifier "platform" <>)) - -(define-dump dump-gene-chip - (tables (GeneChip)) - (schema-triples - (gn:name rdfs:range rdfs:Literal)) - (triples (gene-chip-name->id (field GeneChip Name)) - (set rdf:type 'gn:platform) - (set gn:name (field GeneChip GeneChipName)))) - -;; Dumping Datasets from InfoFiles that don't exist in PublishFreeze and GenoFreeze -(define-dump dump-info-files - (tables (InfoFiles - (left-join PublishFreeze "ON InfoFiles.InfoPageName = PublishFreeze.Name") - (left-join GenoFreeze "ON InfoFiles.InfoPageName = GenoFreeze.Name") - (left-join Datasets "USING (DatasetId)") - (left-join DatasetStatus "USING (DatasetStatusId)") - (left-join Species "USING (SpeciesId)") - (left-join Tissue "USING (TissueId)") - (left-join Investigators "USING (InvestigatorId)") - (left-join AvgMethod "USING (AvgMethodId)") - (left-join GeneChip "USING (GeneChipId)")) - "WHERE GN_AccesionId IS NOT NULL") - (schema-triples - (gn:datasetOfInvestigator rdfs:domain gn:dataset) - (gn:datasetOfInvestigator rdfs:range foaf:Person) - (gn:datasetOfSpecies rdfs:domain gn:dataset) - (gn:datasetOfSpecies rdfs:range gn:species) - (gn:datasetOfInbredSet rdfs:domain gn:dataset) - (gn:datasetOfInbredSet rdfs:range gn:inbredSet) - (gn:datasetOfTissue rdfs:domain gn:dataset) - (gn:datasetOfTissue rdfs:range gn:tissue) - (gn:normalization rdfs:domain gn:dataset) - (gn:normalization rdfs:range gn:avgMethod) - (gn:datasetOfPlatform rdfs:domain gn:dataset) - (gn:datasetOfPlatform rdfs:range gn:geneChip) - (gn:accessionId rdfs:range rdfs:Literal) - (gn:datasetStatusName rdfs:range rdfs:Literal) - (gn:summary rdfs:range rdfs:Literal) - (gn:aboutTissue rdfs:range rdfs:Literal) - (gn:geoSeries rdfs:range rdfs:Literal) - (gn:name rdfs:range rdfs:Literal) - (gn:title rdfs:range rdfs:Literal) - (gn:specifics rdfs:range rdfs:Literal) - (gn:datasetGroup rdfs:range rdfs:Literal) - (gn:aboutCases rdfs:range rdfs:Literal) - (gn:aboutPlatform rdfs:range rdfs:Literal) - (gn:aboutDataProcessing rdfs:range rdfs:Literal) - (gn:notes rdfs:range rdfs:Literal) - (gn:experimentDesign rdfs:range rdfs:Literal) - (gn:contributors rdfs:range rdfs:Literal) - (gn:citation rdfs:range rdfs:Literal) - (gn:acknowledgment rdfs:range rdfs:Literal)) - (triples (string->identifier "dataset" - (field InfoFiles InfoPageName)) - (set rdf:type 'gn:dataset) - (set gn:name (field InfoFiles InfoPageName)) - (set dct:created - (let* ((genotype-create-time - (field ("IFNULL(GenoFreeze.CreateTime, '')" - createTimeGenoFreeze))) - (phenotype-create-time - (field ("IFNULL(PublishFreeze.CreateTime, '')" - createTimePublishFreeze))) - (create-time - (if (string-null? genotype-create-time) - phenotype-create-time genotype-create-time))) - (annotate-field - create-time - '^^xsd:datetime))) - (set gn:datasetOfInvestigator - (investigator-attributes->id (field Investigators FirstName) - (field Investigators LastName) - (field Investigators Email))) - (set gn:accessionId (string-append "GN" (number->string - (field InfoFiles GN_AccesionId)))) - (set gn:datasetStatusName (string-downcase - (field DatasetStatus DatasetStatusName))) - (set gn:datasetOfSpecies (binomial-name->species-id - (field Species FullName BinomialName))) - (set gn:datasetOfTissue (tissue-short-name->id (field Tissue Short_Name))) - (set gn:normalization - (avg-method-name->id - ;; If AvgMethodName is NULL, assume N/A. - (if (string-blank? (field AvgMethod Name AvgMethodName)) - "N/A" (field AvgMethod Name AvgMethodName)))) - (set gn:datasetOfPlatform (gene-chip-name->id (field GeneChip Name GeneChip))) - (set gn:summary - (sanitize-rdf-string (field Datasets Summary))) - (set gn:aboutTissue - (sanitize-rdf-string (field Datasets AboutTissue))) - (set gn:geoSeries - (and (not (string-prefix-ci? "no geo series" - (field Datasets GeoSeries))) - (field Datasets GeoSeries))) - (set gn:title (field InfoFiles Title)) - (set gn:specifics (sanitize-rdf-string (field InfoFiles Specifics))) - (set gn:datasetGroup (field Datasets DatasetName DatasetGroup)) - (set gn:aboutCases (sanitize-rdf-string (field Datasets AboutCases))) - (set gn:aboutPlatform (sanitize-rdf-string (field Datasets AboutPlatform))) - (set gn:aboutDataProcessing (sanitize-rdf-string - (field Datasets AboutDataProcessing))) - (set gn:notes (sanitize-rdf-string (field Datasets Notes))) - (set gn:experimentDesign (sanitize-rdf-string - (field Datasets ExperimentDesign))) - (set gn:contributors (sanitize-rdf-string (field Datasets Contributors))) - (set gn:citation (sanitize-rdf-string (field Datasets Citation))) - (set gn:acknowledgment (sanitize-rdf-string - (field Datasets Acknowledgment))))) - -;; Dumping Phenotypes from PublishFreeze that are not present in the InfoFiles tables -(define-dump dump-phenotypes - (tables (Phenotype - (left-join PublishXRef "ON Phenotype.Id = PublishXRef.PhenotypeId") - (left-join Publication "ON Publication.Id = PublishXRef.PublicationId") - (left-join PublishFreeze "ON PublishFreeze.InbredSetId = PublishXRef.InbredSetId") - (left-join InfoFiles "ON InfoFiles.InfoPageName = PublishFreeze.Name"))) - (schema-triples - (gn:phenotypeDataset rdfs:subPropertyOf gn:dataset)) - (triples (ontology 'phenotype: - (regexp-substitute/global #f "[^A-Za-z0-9:]" - (field ("CONCAT(IF(PublishFreeze.Name IS NULL, '', CONCAT(PublishFreeze.Name, ':')), IF(Phenotype.Post_publication_abbreviation IS NULL, IF(Phenotype.Pre_publication_abbreviation IS NULL, Phenotype.Id, Pre_publication_abbreviation), Phenotype.Post_publication_abbreviation))" abbrev)) - 'pre "_" 'post)) - (set rdf:type 'gn:phenotype) - (set gn:name (sanitize-rdf-string - (field - ("CAST(CONVERT(BINARY CONVERT(CONCAT(IF(PublishFreeze.Name IS NULL, '', CONCAT(PublishFreeze.Name, '-')), IF(Phenotype.Post_publication_abbreviation IS NULL, IF(Phenotype.Pre_publication_abbreviation IS NULL, Phenotype.Id, Phenotype.Pre_publication_abbreviation), Phenotype.Post_publication_abbreviation)) USING latin1) USING utf8) AS VARCHAR(10000))" - abbrev)))) - ;; There is no row with an empty post-publication description so - ;; use this field as the main publication description - (set gn:publicationDescription - (sanitize-rdf-string - (field ("CAST(CONVERT(BINARY CONVERT(Phenotype.Post_publication_description USING latin1) USING utf8) AS CHAR(10000))" - postPubDescr)))) - (set gn:originalDescription (sanitize-rdf-string - (delete-substrings - (field Phenotype Original_description) - "Original post publication description: "))) - (set gn:prePublicationDescription - (sanitize-rdf-string - (field - ("CAST(CONVERT(BINARY CONVERT(Phenotype.Pre_publication_description USING latin1) USING utf8) AS VARCHAR(15000))" - prePubDesc)))) - (set gn:prePublicationAbbreviation (sanitize-rdf-string (field Phenotype Pre_publication_abbreviation))) - (set gn:postPublicationAbbreviation (sanitize-rdf-string (field Phenotype Post_publication_abbreviation))) - (set gn:labCode (field Phenotype Lab_code)) - (set gn:submitter (sanitize-rdf-string (field Phenotype Submitter))) - (set gn:owner (sanitize-rdf-string (field Phenotype Owner))) - (set gn:mean (annotate-field (field ("IFNULL(PublishXRef.mean, '')" mean)) - '^^xsd:float)) - (set gn:locus (field PublishXRef Locus)) - (set gn:LRS (annotate-field (field ("IFNULL(PublishXRef.LRS, '')" lrs)) '^^xsd:float)) - (set gn:additive (annotate-field (field ("IFNULL(PublishXRef.additive, '')" additive)) '^^xsd:decimal)) - (set gn:sequence (annotate-field (field PublishXRef Sequence) '^^xsd:int)) - (set gn:phenotypeOfDataset (string->identifier "dataset" (field PublishFreeze Name))) - (set gn:phenotypeOfPublication - (let ((pmid (field - ("IF(Publication.PubMed_ID IS NULL, '', CONVERT(Publication.PubMed_Id, INT))" - pmid))) - (publication-id (field Publication Id))) - (if (string-null? pmid) - (string->identifier "publication" - (number->string publication-id)) - (ontology 'pubmed: pmid)))))) - -(define-dump dump-genotypes - (tables (GenoFreeze - (left-join InfoFiles "ON InfoFiles.InfoPageName = GenoFreeze.Name")) - "WHERE GenoFreeze.Name NOT IN (SELECT DISTINCT InfoFiles.InfoPageName FROM InfoFiles)") - (schema-triples - (gn:phenotypeDataset rdfs:subPropertyOf gn:dataset)) - (triples (string->identifier "dataset" - (field GenoFreeze Name)) - (set rdf:type 'gn:genotypeDataset) - (set gn:name (field GenoFreeze FullName)) - (set dct:created (annotate-field - (field GenoFreeze CreateTime) - '^^xsd:datetime)))) - - -(define (dump-data-table db table-name data-field) - (let ((dump-directory (string-append %dump-directory "/" table-name)) - (port #f) - (current-strain-id #f)) - (unless (file-exists? dump-directory) - (mkdir dump-directory)) - (sql-for-each (match-lambda - (((_ . strain-id) - (_ . value)) - ;; Close file if new strain. - (when (and port - (not (= current-strain-id strain-id))) - (close-port port) - (set! port #f)) - ;; If no file is open, open new file. - (unless port - (set! current-strain-id strain-id) - (let ((filename (string-append dump-directory - "/" (number->string strain-id)))) - (display filename (current-error-port)) - (newline (current-error-port)) - (set! port (open-output-file filename)))) - (display value port) - (newline port))) - db - (format #f "SELECT StrainId, ~a FROM ~a ORDER BY StrainId" - data-field table-name)) - (close-port port))) - - -;;; Dump schema - -(define (tables db) - "Return list of all tables in DB. Each element of the returned list -is a <table> object." - (map (lambda (table) - (set-table-columns table - (sql-map (lambda (row) - (make-column (assoc-ref row "Field") - (assoc-ref row "Type"))) - db - (format #f "SHOW COLUMNS FROM ~a" (table-name table))))) - (sql-map (lambda (row) - (make-table (assoc-ref row "table_name") - ;; FIXME: This is probably correct only for - ;; MyISAM tables. - (assoc-ref row "data_length") - #f)) - db - (select-query ((information_schema.tables table_name) - (information_schema.tables data_length)) - (information_schema.tables) - (format #f "WHERE table_schema = '~a'" - (assq-ref %connection-settings 'sql-database)))))) - -(define (dump-schema db) - (let ((tables (tables db))) - (for-each (lambda (table) - (let ((table-id (string->identifier - "table" - ;; We downcase table names in - ;; identifiers. So, we distinguish - ;; between the user and User tables. - (if (string=? (table-name table) "User") - "user2" - (table-name table))))) - (triple table-id 'rdf:type 'gn:sqlTable) - (triple table-id 'gn:name (table-name table)) - (triple table-id 'gn:hasSize (table-size table)) - (for-each (lambda (column) - (let ((column-id (column-id (table-name table) - (column-name column)))) - (triple column-id 'rdf:type 'gn:sqlTableField) - (triple column-id 'gn:name (column-name column)) - (triple column-id 'gn:sqlFieldType (column-type column)) - (triple table-id 'gn:hasField column-id))) - (table-columns table)))) - tables))) - -(define-dump dump-groups - (tables (InbredSet - (left-join Species "USING (SpeciesId)")) - (string-join - '("WHERE Species.Name IN " - "(SELECT Name FROM Species ORDER BY OrderId) " - "GROUP BY InbredSet.Name " - "ORDER BY " - "IFNULL(InbredSet.FamilyOrder, InbredSet.FullName) " - "ASC, IFNULL(InbredSet.Family, InbredSet.FullName) " - "ASC, InbredSet.FullName ASC, InbredSet.MenuOrderId ASC"))) - (schema-triples - (gn:name rdfs:range rdfs:Literal) - (gn:binomialName rdfs:range rdfs:Literal) - (gn:species rdfs:range gn:species)) - (triples (string->identifier "inbredSet" (field InbredSet Name)) - (set gn:name (field InbredSet Name)) - (set gn:binomialName (field InbredSet fullName)) - (set gn:species (field Species Name)))) - -(define-dump dump-genewiki-symbols - (tables (GeneRIF_BASIC - (left-join Species "USING (SpeciesId)")) - "GROUP BY GeneId ORDER BY BINARY symbol") - (schema-triples - (gn:symbol rdfs:domain gn:geneWikiEntry) - (gn:wikiEntryOfSpecies rdfs:range gn:species) - (gn:taxid rdfs:domain gn:geneWikiEntry)) - (triples (ontology 'generif: (field GeneRIF_BASIC GeneId)) - (multiset gn:symbol (string-split (field ("GROUP_CONCAT(DISTINCT symbol)" symbol)) - #\,)) - (multiset gn:wikiEntryOfSpecies - (string-split - (field ("GROUP_CONCAT(DISTINCT Species.SpeciesName)" species)) - #\,)) - (multiset gn:taxId (map (cut ontology 'taxon: <>) - (string-split (field ("GROUP_CONCAT(DISTINCT TaxID)" taxId)) - #\,))))) -;; GeneRIF metadata -(define-dump dump-gn-genewiki-entries - (tables (GeneRIF - (left-join GeneRIF_BASIC "USING (symbol)") - (left-join Species "ON Species.SpeciesId = GeneRIF.SpeciesId") - (left-join GeneRIFXRef "ON GeneRIFXRef.GeneRIFId = GeneRIF.Id") - (left-join GeneCategory "ON GeneRIFXRef.GeneCategoryId = GeneCategory.Id")) - "WHERE GeneRIF.display > 0 AND GeneRIF.VersionId = 0 GROUP BY GeneRIF.symbol") - (schema-triples - (gn:geneWikiEntry a rdfs:Class) - (gn:geneWikiEntry a owl:Class) - (gn:geneWikiEntry rdfs:comment "Represents GeneRIF Entries") - (gn:geneCategory rdfs:domain gn:geneWikiEntry) - (gn:geneWikiEntryOfGn rdfs:domain gn:geneWikiEntry) - (gn:geneWikiEntry rdfs:domain gn:geneWikiEntry)) - (triples - (let ([geneid (field GeneRIF_BASIC GeneId)]) - (if (eq? geneid 0) - (ontology 'gn:anonSymbol_ - (field GeneRIF symbol)) - (ontology 'generif: - geneid))) - (set rdf:type (if (eq? (field GeneRIF_BASIC GeneId) 0) - 'gn:geneWikiEntry - "")) - (set gn:wikiEntryOfSpecies (if (eq? (field GeneRIF_BASIC GeneId) 0) - (field Species SpeciesName) - "")) - ;; This only dumps symbols not present in the GeneRIF_BASIC table - (set gn:symbol (let ([geneid (field GeneRIF_BASIC GeneId)]) - (if (eq? geneid 0) - (field GeneRIF symbol) - ""))) - (multiset gn:geneWikiEntryOfGn - (let* ([entries - (replace-substrings - (field - ("GROUP_CONCAT(DISTINCT CONCAT_WS('::::', IFNULL(GeneCategory.Name, ''), IFNULL(GeneRIF.PubMed_ID, ''), GeneRIF.email, GeneRIF.comment, GeneRIF.createtime, IFNULL(weburl, '')) SEPARATOR';;;;;')" - wikientry)) - '(("\x28" . "") - ("\x29" . "") - ("\xa0" . " ") - ("â\x81„" . "/") - ("â€\x9d" . #\") - ("’" . #\') - ("\x02" . "") - ("\x01" . "") - ("β" . "β") - ("α-Â\xad" . "α") - ("Â\xad" . "") - ("α" . "α") - ("–" . "-")))] - [comments (string-split-substring entries ";;;;;")]) - (map - (match-lambda - ((genecategory pmid email text createtime weburl) - (blank-node - (set gn:geneCategory genecategory) - (multiset dct:source - (map (lambda (el) (if (string-null? el) - "" - (ontology 'pubmed: el))) - (string-split pmid #\space))) - (set dct:creator (regexp-substitute/global #f "@.*$" - email - 'pre - "" - 'post)) - (set gn:geneWikiEntry - (annotate-field text '^^xsd:string)) - (set dct:created (annotate-field - createtime - '^^xsd:datetime)) - (set foaf:homepage weburl)))) - (map - (cut string-split-substring <> "::::") - comments)))))) - -(define-dump dump-ncbi-genewiki-entries - (tables (GeneRIF_BASIC) - "GROUP BY GeneId, comment, createtime") - (schema-triples - (gn:geneWikiEntryofNCBI rdfs:domain gn:geneWikiEntry)) - (triples (ontology 'generif: - (field GeneRIF_BASIC GeneId)) - (set gn:geneWikiEntryOfNCBI - (blank-node - (set gn:geneWikiEntry - (annotate-field (field GeneRIF_BASIC comment) - '^^xsd:string)) - (multiset dct:source (map (lambda (el) (if (string-null? el) - "" - (ontology 'pubmed: el))) - (string-split (field ("GROUP_CONCAT(PubMed_ID)" pmids)) - #\,))) - (set dct:created (annotate-field (time-unix->string - (field GeneRIF_BASIC createtime) "~5") - '^^xsd:datetime)))))) - - -;; Main function - -(call-with-genenetwork-database - (lambda (db) - (with-output-to-file (string-append %dump-directory "/dump.ttl") - (lambda () - (prefix "rdf:" "<http://www.w3.org/1999/02/22-rdf-syntax-ns#>") - (prefix "rdfs:" "<http://www.w3.org/2000/01/rdf-schema#>") - (prefix "foaf:" "<http://xmlns.com/foaf/0.1/>") - (prefix "gn:" "<http://genenetwork.org/>") - (prefix "dct:" "<http://purl.org/dc/terms/>") - (prefix "pubmed:" "<http://rdf.ncbi.nlm.nih.gov/pubmed/>") - (prefix "up:" "<http://purl.uniprot.org/core/>") - (prefix "taxon:" "<http://purl.uniprot.org/taxonomy/>") - (prefix "generif:" "<http://www.ncbi.nlm.nih.gov/gene?cmd=Retrieve&dopt=Graphics&list_uids=>") - (prefix "xsd:" "<http://www.w3.org/2001/XMLSchema#>") - (prefix "owl:" "<http://www.w3.org/2002/07/owl#>") - (prefix "phenotype:" "<http://genenetwork.org/phenotype/>") - (newline) - (dump-genewiki-symbols db) - (dump-gn-genewiki-entries db) - (dump-ncbi-genewiki-entries db) - (dump-species db) - (dump-strain db) - (dump-mapping-method db) - (dump-inbred-set db) - (dump-publishfreeze db) - (dump-publication db) - (dump-tissue db) - (dump-investigators db) - (dump-avg-method db) - (dump-gene-chip db) - (dump-info-files db) - (dump-schema db) - (dump-groups db) - (dump-published-phenotypes db))))) - diff --git a/examples/dump-dataset-metadata.scm b/examples/dump-dataset-metadata.scm new file mode 100755 index 0000000..aa7a5f2 --- /dev/null +++ b/examples/dump-dataset-metadata.scm @@ -0,0 +1,220 @@ +#! /usr/bin/env guile +!# + +(use-modules (srfi srfi-1) + (srfi srfi-26) + (ice-9 match) + (ice-9 regex) + (dump strings) + (dump sql) + (dump triples) + (dump special-forms)) + + + +(define %connection-settings + (call-with-input-file (list-ref (command-line) 1) + read)) + +(define %dump-directory + (list-ref (command-line) 2)) + + + +;; One email ID in the Investigators table has spaces in it. This +;; function fixes that. +(define (fix-email-id email) + (string-delete #\space email)) + +(define (investigator-attributes->id first-name last-name email) + ;; There is just one record corresponding to "Evan Williams" which + ;; does not have an email ID. To accommodate that record, we + ;; construct the investigator ID from not just the email ID, but + ;; also the first and the last names. It would be preferable to just + ;; find Evan Williams' email ID and insert it into the database. + (string->identifier "investigator" + (string-join + ;; Add special case for Yohan Bossé whose name + ;; has unprintable characters. + ;; TODO: Fix Yohan Bossé's name in the database. + (let ((last-name (if (string=? first-name "Yohan") + "Bosse" + last-name))) + (list first-name last-name (fix-email-id email))) + "_"))) + +(define-dump dump-investigators + ;; There are a few duplicate entries. We group by email to + ;; deduplicate. + (tables (Investigators) + "GROUP BY Email") + (schema-triples + ;; TODO: Are ranges required for FOAF predicates? Can they not be + ;; obtained from the FOAF spec? + (foaf:name rdfs:range rdfs:Literal) + (foaf:givenName rdfs:range rdfs:Literal) + (foaf:familyName rdfs:range rdfs:Literal) + (foaf:phone rdfs:range rdfs:Literal) + (foaf:mbox rdfs:range rdfs:Literal) + (foaf:homepage rdfs:range rdfs:Literal) + (gn:address rdfs:range rdfs:Literal) + (gn:city rdfs:range rdfs:Literal) + (gn:state rdfs:range rdfs:Literal) + (gn:zipCode rdfs:range rdfs:Literal) + (gn:country rdfs:range rdfs:Literal)) + (triples (investigator-attributes->id (field Investigators FirstName) + (field Investigators LastName) + (field Investigators Email)) + (set rdf:type 'foaf:Person) + ;; Special case Yohan Bossé's name since the last name has + ;; unprintable characters. + (set foaf:name (string-append (field Investigators FirstName) " " + (if (string=? (field Investigators FirstName) "Yohan") + "Bossé" + (field Investigators LastName)))) + (set foaf:givenName (field Investigators FirstName)) + ;; Special case Yohan Bossé's name since the last name has + ;; unprintable characters. + (set foaf:familyName (if (string=? (field Investigators FirstName) "Yohan") + "Bossé" + (field Investigators LastName))) + (set foaf:phone (field Investigators Phone)) + (set foaf:mbox (fix-email-id (field Investigators Email))) + (set foaf:homepage (field Investigators Url)) + (set gn:address (field Investigators Address)) + (set gn:city (field Investigators City)) + (set gn:state (field Investigators State)) + (set gn:zipCode (field Investigators ZipCode)) + (set gn:country (field Investigators Country)))) + +(define-dump dump-info-files + (tables (InfoFiles + (left-join PublishFreeze "ON InfoFiles.InfoPageName = PublishFreeze.Name") + (left-join GenoFreeze "ON InfoFiles.InfoPageName = GenoFreeze.Name") + (left-join ProbeSetFreeze "ON InfoFiles.InfoPageName = ProbeSetFreeze.Name") + (left-join Datasets "USING (DatasetId)") + (left-join DatasetStatus "USING (DatasetStatusId)") + (left-join Species "USING (SpeciesId)") + (left-join Tissue "USING (TissueId)") + (left-join Investigators "USING (InvestigatorId)") + (left-join AvgMethod "USING (AvgMethodId)") + (left-join GeneChip "USING (GeneChipId)")) + "WHERE GN_AccesionId IS NOT NULL") + (schema-triples + (gn:datasetOfInvestigator rdfs:domain gn:dataset) + (gn:datasetOfInvestigator rdfs:range foaf:Person) + (gn:datasetOfSpecies rdfs:domain gn:dataset) + (gn:datasetOfSpecies rdfs:range gn:species) + (gn:datasetOfInbredSet rdfs:domain gn:dataset) + (gn:datasetOfInbredSet rdfs:range gn:inbredSet) + (gn:datasetOfTissue rdfs:domain gn:dataset) + (gn:datasetOfTissue rdfs:range gn:tissue) + (gn:normalization rdfs:domain gn:dataset) + (gn:normalization rdfs:range gn:avgMethod) + (gn:datasetOfPlatform rdfs:domain gn:dataset) + (gn:datasetOfPlatform rdfs:range gn:geneChip) + (gn:accessionId rdfs:range rdfs:Literal) + (gn:datasetStatusName rdfs:range rdfs:Literal) + (gn:summary rdfs:range rdfs:Literal) + (gn:aboutTissue rdfs:range rdfs:Literal) + (gn:geoSeries rdfs:range rdfs:Literal) + (gn:name rdfs:range rdfs:Literal) + (gn:title rdfs:range rdfs:Literal) + (gn:specifics rdfs:range rdfs:Literal) + (gn:datasetGroup rdfs:range rdfs:Literal) + (gn:aboutCases rdfs:range rdfs:Literal) + (gn:aboutPlatform rdfs:range rdfs:Literal) + (gn:aboutDataProcessing rdfs:range rdfs:Literal) + (gn:notes rdfs:range rdfs:Literal) + (gn:experimentDesign rdfs:range rdfs:Literal) + (gn:contributors rdfs:range rdfs:Literal) + (gn:citation rdfs:range rdfs:Literal) + (gn:acknowledgment rdfs:range rdfs:Literal)) + (triples (string->identifier "dataset" + (field InfoFiles InfoPageName)) + ;; Add GeneChipName and GeoPlatform: + ;; GeneChip.GeneChipName AS gene_chip_name + ;; GeneChip.GeoPlatform AS geo_platform + (set rdf:type (string->symbol + (field ("IF(GenoFreeze.Id IS NOT NULL, 'gn:genotypeDataset', IF(PublishFreeze.Id IS NOT NULL, 'gn:phenotypeDataset', 'gn:dataset'))" + rdfType)))) + (set gn:name (field InfoFiles InfoPageName)) + (set dct:created + (field ("IFNULL(GenoFreeze.CreateTime, IFNULL(PublishFreeze.CreateTime, IFNULL(ProbeSetFreeze.CreateTime, '')))" + createTimeGenoFreeze))) + (set gn:datasetOfInvestigator + (investigator-attributes->id (field Investigators FirstName) + (field Investigators LastName) + (field Investigators Email))) + (set gn:accessionId (string-append "GN" (number->string + (field InfoFiles GN_AccesionId)))) + (set gn:datasetStatusName (string-downcase + (field DatasetStatus DatasetStatusName))) + (set gn:datasetOfSpecies (string->identifier "species" + (field Species FullName BinomialName))) + (set gn:datasetOfTissue (string->identifier "tissue" + (field Tissue Short_Name))) + (set gn:normalization + (string->identifier "avgmethod" + ;; If AvgMethodName is NULL, assume N/A. + (if (string-blank? (field AvgMethod Name AvgMethodName)) + "N/A" (field AvgMethod Name AvgMethodName)))) + (set gn:datasetOfPlatform + (string->identifier "platform" + (field GeneChip Name GeneChip))) + (set gn:summary + (sanitize-rdf-string (field Datasets Summary))) + (set gn:aboutTissue + (sanitize-rdf-string (field Datasets AboutTissue))) + (set gn:geoSeries + (and (not (string-prefix-ci? "no geo series" + (field Datasets GeoSeries))) + (field Datasets GeoSeries))) + (set gn:title (field InfoFiles Title)) + (set gn:specifics (sanitize-rdf-string (field InfoFiles Specifics))) + (set gn:datasetGroup (field Datasets DatasetName DatasetGroup)) + (set gn:aboutCases (sanitize-rdf-string (field Datasets AboutCases))) + (set gn:aboutPlatform (sanitize-rdf-string (field Datasets AboutPlatform))) + (set gn:aboutDataProcessing (sanitize-rdf-string + (field Datasets AboutDataProcessing))) + (set gn:notes (sanitize-rdf-string (field Datasets Notes))) + (set gn:experimentDesign (sanitize-rdf-string + (field Datasets ExperimentDesign))) + (set gn:contributors (sanitize-rdf-string (field Datasets Contributors))) + (set gn:citation (sanitize-rdf-string (field Datasets Citation))) + (set gn:acknowledgment (sanitize-rdf-string + (field Datasets Acknowledgment))))) + + + + +(call-with-target-database + %connection-settings + (lambda (db) + (with-output-to-file (string-append %dump-directory "dump-info-pages.ttl") + (lambda () + (prefix "chebi:" "<http://purl.obolibrary.org/obo/CHEBI_>") + (prefix "dct:" "<http://purl.org/dc/terms/>") + (prefix "foaf:" "<http://xmlns.com/foaf/0.1/>") + (prefix "generif:" "<http://www.ncbi.nlm.nih.gov/gene?cmd=Retrieve&dopt=Graphics&list_uids=>") + (prefix "gn:" "<http://genenetwork.org/>") + (prefix "hgnc:" "<http://bio2rdf.org/hgnc:>") + (prefix "homologene:" "<https://bio2rdf.org/homologene:>") + (prefix "kegg:" "<http://bio2rdf.org/ns/kegg#>") + (prefix "molecularTrait:" "<http://genenetwork.org/molecular-trait/>") + (prefix "nuccore:" "<https://www.ncbi.nlm.nih.gov/nuccore/>") + (prefix "omim:" "<https://www.omim.org/entry/>") + (prefix "owl:" "<http://www.w3.org/2002/07/owl#>") + (prefix "phenotype:" "<http://genenetwork.org/phenotype/>") + (prefix "pubchem:" "<https://pubchem.ncbi.nlm.nih.gov/>") + (prefix "pubmed:" "<http://rdf.ncbi.nlm.nih.gov/pubmed/>") + (prefix "rdf:" "<http://www.w3.org/1999/02/22-rdf-syntax-ns#>") + (prefix "rdfs:" "<http://www.w3.org/2000/01/rdf-schema#>") + (prefix "taxon:" "<http://purl.uniprot.org/taxonomy/>") + (prefix "uniprot:" "<http://purl.uniprot.org/uniprot/>") + (prefix "up:" "<http://purl.uniprot.org/core/>") + (prefix "xsd:" "<http://www.w3.org/2001/XMLSchema#>") + (newline) + (dump-info-files db) + (dump-investigators db)) + #:encoding "utf8"))) diff --git a/examples/dump-generif.scm b/examples/dump-generif.scm new file mode 100755 index 0000000..415cb67 --- /dev/null +++ b/examples/dump-generif.scm @@ -0,0 +1,159 @@ +#! /usr/bin/env guile +!# + +(use-modules (srfi srfi-1) + (srfi srfi-26) + (ice-9 match) + (ice-9 regex) + (dump strings) + (dump sql) + (dump triples) + (dump special-forms)) + + + +(define %connection-settings + (call-with-input-file (list-ref (command-line) 1) + read)) + +(define %dump-directory + (list-ref (command-line) 2)) + + + +(define-dump dump-genewiki-symbols + (tables (GeneRIF_BASIC + (left-join Species "USING (SpeciesId)")) + "GROUP BY GeneId ORDER BY BINARY symbol") + (schema-triples + (gn:symbol rdfs:domain gn:geneWikiEntry) + (gn:wikiEntryOfSpecies rdfs:range gn:species) + (gn:taxid rdfs:domain gn:geneWikiEntry)) + (triples (ontology 'generif: (field GeneRIF_BASIC GeneId)) + (multiset gn:symbol (string-split (field ("GROUP_CONCAT(DISTINCT symbol)" symbol)) + #\,)) + (multiset gn:wikiEntryOfSpecies + (string-split + (field ("GROUP_CONCAT(DISTINCT Species.SpeciesName)" species)) + #\,)) + (multiset gn:taxId (map (cut ontology 'taxon: <>) + (string-split (field ("GROUP_CONCAT(DISTINCT TaxID)" taxId)) + #\,))))) + +(define-dump dump-gn-genewiki-entries + (tables (GeneRIF + (left-join GeneRIF_BASIC "USING (symbol)") + (left-join Species "ON Species.SpeciesId = GeneRIF.SpeciesId") + (left-join GeneRIFXRef "ON GeneRIFXRef.GeneRIFId = GeneRIF.Id") + (left-join GeneCategory "ON GeneRIFXRef.GeneCategoryId = GeneCategory.Id")) + "WHERE GeneRIF.display > 0 AND GeneRIF.VersionId = 0 GROUP BY GeneRIF.symbol") + (schema-triples + (gn:geneWikiEntry a rdfs:Class) + (gn:geneWikiEntry a owl:Class) + (gn:geneWikiEntry rdfs:comment "Represents GeneRIF Entries") + (gn:geneCategory rdfs:domain gn:geneWikiEntry) + (gn:geneWikiEntryOfGn rdfs:domain gn:geneWikiEntry) + (gn:geneWikiEntry rdfs:domain gn:geneWikiEntry)) + (triples + (let ([geneid (field GeneRIF_BASIC GeneId)]) + (if (eq? geneid 0) + (ontology 'gn:anonSymbol_ + (field GeneRIF symbol)) + (ontology 'generif: + geneid))) + (set rdf:type + (if (string-null? (field ("IFNULL(GeneRIF_BASIC.GeneId, '')" geneWikiEntryP))) + "" + 'gn:geneWikiEntry)) + (set gn:wikiEntryOfSpecies + (field Species SpeciesName)) + ;; This only dumps symbols not present in the GeneRIF_BASIC table + (set gn:symbol (let ([geneid (field GeneRIF_BASIC GeneId)]) + (if (eq? geneid 0) + (field GeneRIF symbol) + ""))) + (multiset gn:geneWikiEntryOfGn + (let* ([entries + (field + ("GROUP_CONCAT(DISTINCT CONCAT_WS('::::', IFNULL(GeneCategory.Name, ''), IFNULL(GeneRIF.PubMed_ID, ''), GeneRIF.email, CAST(CONVERT(BINARY CONVERT(GeneRIF.comment USING latin1) USING utf8) AS VARCHAR(15000)), GeneRIF.createtime, IFNULL(weburl, '')) SEPARATOR';;;;;')" + wikientry))] + [comments (string-split-substring entries ";;;;;")]) + (map + (match-lambda + ((genecategory pmid email text createtime weburl) + (blank-node + (set gn:geneCategory genecategory) + (multiset dct:source + (map (lambda (el) (if (string-null? el) + "" + (ontology 'pubmed: el))) + (string-split pmid #\space))) + (set dct:creator (regexp-substitute/global #f "@.*$" + email + 'pre + "" + 'post)) + (set gn:geneWikiEntry + (annotate-field text '^^xsd:string)) + (set dct:created (annotate-field + createtime + '^^xsd:datetime)) + (set foaf:homepage weburl)))) + (map + (cut string-split-substring <> "::::") + comments)))))) + +(define-dump dump-ncbi-genewiki-entries + (tables (GeneRIF_BASIC) + "GROUP BY GeneId, comment, createtime") + (schema-triples + (gn:geneWikiEntryofNCBI rdfs:domain gn:geneWikiEntry)) + (triples (ontology 'generif: + (field GeneRIF_BASIC GeneId)) + (set gn:geneWikiEntryOfNCBI + (blank-node + (set gn:geneWikiEntry + (annotate-field (field GeneRIF_BASIC comment) + '^^xsd:string)) + (multiset dct:source (map (lambda (el) (if (string-null? el) + "" + (ontology 'pubmed: el))) + (string-split (field ("GROUP_CONCAT(PubMed_ID)" pmids)) + #\,))) + (set dct:created (annotate-field (time-unix->string + (field GeneRIF_BASIC createtime) "~5") + '^^xsd:datetime)))))) + + + +(call-with-target-database + %connection-settings + (lambda (db) + (with-output-to-file (string-append %dump-directory "dump-generif.ttl") + (lambda () + (prefix "rdf:" "<http://www.w3.org/1999/02/22-rdf-syntax-ns#>") + (prefix "rdfs:" "<http://www.w3.org/2000/01/rdf-schema#>") + (prefix "foaf:" "<http://xmlns.com/foaf/0.1/>") + (prefix "gn:" "<http://genenetwork.org/>") + (prefix "dct:" "<http://purl.org/dc/terms/>") + (prefix "pubmed:" "<http://rdf.ncbi.nlm.nih.gov/pubmed/>") + (prefix "up:" "<http://purl.uniprot.org/core/>") + (prefix "taxon:" "<http://purl.uniprot.org/taxonomy/>") + (prefix "generif:" "<http://www.ncbi.nlm.nih.gov/gene?cmd=Retrieve&dopt=Graphics&list_uids=>") + (prefix "xsd:" "<http://www.w3.org/2001/XMLSchema#>") + (prefix "owl:" "<http://www.w3.org/2002/07/owl#>") + (prefix "phenotype:" "<http://genenetwork.org/phenotype/>") + (prefix "molecularTrait:" "<http://genenetwork.org/molecular-trait/>") + (prefix "nuccore:" "<https://www.ncbi.nlm.nih.gov/nuccore/>") + (prefix "omim:" "<https://www.omim.org/entry/>") + (prefix "pubchem:" "<https://pubchem.ncbi.nlm.nih.gov/>") + (prefix "uniprot:" "<http://purl.uniprot.org/uniprot/>") + (prefix "hgnc:" "<http://bio2rdf.org/hgnc:>") + (prefix "homologene:" "<https://bio2rdf.org/homologene:>") + (prefix "chebi:" "<http://purl.obolibrary.org/obo/CHEBI_>") + (prefix "kegg:" "<http://bio2rdf.org/ns/kegg#>") + (newline) + (dump-genewiki-symbols db) + (dump-gn-genewiki-entries db) + (dump-ncbi-genewiki-entries db)) + #:encoding "utf8"))) diff --git a/examples/dump-phenotype.scm b/examples/dump-phenotype.scm new file mode 100755 index 0000000..842e7e8 --- /dev/null +++ b/examples/dump-phenotype.scm @@ -0,0 +1,131 @@ +#! /usr/bin/env guile +!# + +(use-modules (rnrs programs) + (rnrs io ports) + (srfi srfi-1) + (srfi srfi-26) + (ice-9 match) + (ice-9 regex) + (dump strings) + (dump sql) + (dump triples) + (dump special-forms)) + + + +(define %connection-settings + (call-with-input-file (list-ref (command-line) 1) + read)) + +(define %dump-directory + (list-ref (command-line) 2)) + + + +;; Only dump publish freeze entries that were not dumped from the InfoFiles page +(define-dump dump-publishfreeze + (tables (PublishFreeze + (left-join InfoFiles "ON InfoFiles.InfoPageName = PublishFreeze.Name") + (left-join InbredSet "ON PublishFreeze.InbredSetId = InbredSet.InbredSetId")) + "WHERE PublishFreeze.public > 0 AND InfoFiles.InfoPageName IS NULL") + (schema-triples + (gn:datasetOfInbredSet rdfs:range gn:inbredSet) + (gn:name rdfs:range rdfs:Literal) + (gn:fullName rdfs:range rdfs:Literal) + (gn:shortName rdfs:range rdfs:Literal) + (gn:createTime rdfs:range rdfs:Literal)) + (triples (string->identifier "dataset" (field PublishFreeze Name)) + (set rdf:type 'gn:phenotypeDataset) + (set gn:name (field PublishFreeze Name)) + (set gn:fullName (field PublishFreeze FullName)) + (set gn:shortName (field PublishFreeze ShortName)) + (set gn:createTime (field PublishFreeze CreateTime)) + (set gn:datasetOfInbredSet + (string->identifier "inbredSet" (field InbredSet Name InbredSetName))))) + +(define-dump dump-phenotypes + (tables (Phenotype + (left-join PublishXRef "ON Phenotype.Id = PublishXRef.PhenotypeId") + (left-join Publication "ON Publication.Id = PublishXRef.PublicationId") + (left-join PublishFreeze "ON PublishFreeze.InbredSetId = PublishXRef.InbredSetId") + (left-join InfoFiles "ON InfoFiles.InfoPageName = PublishFreeze.Name"))) + (schema-triples + (gn:phenotypeDataset rdfs:subPropertyOf gn:dataset)) + (triples (ontology 'phenotype: + (regexp-substitute/global #f "[^A-Za-z0-9:]" + (field ("CONCAT(IF(PublishFreeze.Name IS NULL, '', CONCAT(PublishFreeze.Name, ':')), IF(Phenotype.Post_publication_abbreviation IS NULL, IF(Phenotype.Pre_publication_abbreviation IS NULL, Phenotype.Id, Pre_publication_abbreviation), Phenotype.Post_publication_abbreviation))" abbrev)) + 'pre "_" 'post)) + (set rdf:type 'gn:phenotype) + (set gn:name (sanitize-rdf-string + (field + ("CAST(CONVERT(BINARY CONVERT(CONCAT(IF(PublishFreeze.Name IS NULL, '', CONCAT(PublishFreeze.Name, '-')), IF(Phenotype.Post_publication_abbreviation IS NULL, IF(Phenotype.Pre_publication_abbreviation IS NULL, Phenotype.Id, Phenotype.Pre_publication_abbreviation), Phenotype.Post_publication_abbreviation)) USING latin1) USING utf8) AS VARCHAR(10000))" + abbrev)))) + ;; There is no row with an empty post-publication description so + ;; use this field as the main publication description + (set gn:publicationDescription + (sanitize-rdf-string + (field ("CAST(CONVERT(BINARY CONVERT(Phenotype.Post_publication_description USING latin1) USING utf8) AS CHAR(10000))" + postPubDescr)))) + (set gn:originalDescription (sanitize-rdf-string + (delete-substrings + (field Phenotype Original_description) + "Original post publication description: "))) + (set gn:prePublicationDescription + (sanitize-rdf-string + (field + ("CAST(CONVERT(BINARY CONVERT(Phenotype.Pre_publication_description USING latin1) USING utf8) AS VARCHAR(15000))" + prePubDesc)))) + (set gn:prePublicationAbbreviation (sanitize-rdf-string (field Phenotype Pre_publication_abbreviation))) + (set gn:postPublicationAbbreviation (sanitize-rdf-string (field Phenotype Post_publication_abbreviation))) + (set gn:labCode (field Phenotype Lab_code)) + (set gn:submitter (sanitize-rdf-string (field Phenotype Submitter))) + (set gn:owner (sanitize-rdf-string (field Phenotype Owner))) + (set gn:mean (annotate-field (field ("IFNULL(PublishXRef.mean, '')" mean)) + '^^xsd:double)) + (set gn:locus (field PublishXRef Locus)) + (set gn:LRS (annotate-field (field ("IFNULL(PublishXRef.LRS, '')" lrs)) '^^xsd:float)) + (set gn:additive (annotate-field (field ("IFNULL(PublishXRef.additive, '')" additive)) '^^xsd:decimal)) + (set gn:sequence (annotate-field (field PublishXRef Sequence) '^^xsd:int)) + (set gn:phenotypeOfDataset (string->identifier "dataset" (field PublishFreeze Name))) + (set gn:phenotypeOfPublication + (let ((pmid (field + ("IF(Publication.PubMed_ID IS NULL, '', CONVERT(Publication.PubMed_Id, INT))" + pmid))) + (publication-id (field Publication Id))) + (if (string-null? pmid) + (string->identifier "publication" + (number->string publication-id)) + (ontology 'pubmed: pmid)))))) + + +(call-with-target-database + %connection-settings + (lambda (db) + (with-output-to-file (string-append %dump-directory "dump-phenotype.ttl") + (lambda () + (prefix "chebi:" "<http://purl.obolibrary.org/obo/CHEBI_>") + (prefix "dct:" "<http://purl.org/dc/terms/>") + (prefix "foaf:" "<http://xmlns.com/foaf/0.1/>") + (prefix "generif:" "<http://www.ncbi.nlm.nih.gov/gene?cmd=Retrieve&dopt=Graphics&list_uids=>") + (prefix "gn:" "<http://genenetwork.org/>") + (prefix "hgnc:" "<http://bio2rdf.org/hgnc:>") + (prefix "homologene:" "<https://bio2rdf.org/homologene:>") + (prefix "kegg:" "<http://bio2rdf.org/ns/kegg#>") + (prefix "molecularTrait:" "<http://genenetwork.org/molecular-trait/>") + (prefix "nuccore:" "<https://www.ncbi.nlm.nih.gov/nuccore/>") + (prefix "omim:" "<https://www.omim.org/entry/>") + (prefix "owl:" "<http://www.w3.org/2002/07/owl#>") + (prefix "phenotype:" "<http://genenetwork.org/phenotype/>") + (prefix "pubchem:" "<https://pubchem.ncbi.nlm.nih.gov/>") + (prefix "pubmed:" "<http://rdf.ncbi.nlm.nih.gov/pubmed/>") + (prefix "rdf:" "<http://www.w3.org/1999/02/22-rdf-syntax-ns#>") + (prefix "rdfs:" "<http://www.w3.org/2000/01/rdf-schema#>") + (prefix "taxon:" "<http://purl.uniprot.org/taxonomy/>") + (prefix "uniprot:" "<http://purl.uniprot.org/uniprot/>") + (prefix "up:" "<http://purl.uniprot.org/core/>") + (prefix "xsd:" "<http://www.w3.org/2001/XMLSchema#>") + (newline) + (dump-publishfreeze db) + (dump-phenotypes db)) + #:encoding "utf8"))) diff --git a/examples/dump-probeset.scm b/examples/dump-probeset.scm new file mode 100755 index 0000000..d1ea2ae --- /dev/null +++ b/examples/dump-probeset.scm @@ -0,0 +1,213 @@ +#! /usr/bin/env guile +!# + +(use-modules (srfi srfi-1) + (srfi srfi-26) + (ice-9 match) + (ice-9 regex) + (dump strings) + (dump sql) + (dump triples) + (dump special-forms)) + + + +(define %connection-settings + (call-with-input-file (list-ref (command-line) 1) + read)) + +(define %dump-directory + (list-ref (command-line) 2)) + + +(define-dump dump-gene-chip + (tables (GeneChip)) + (schema-triples + (gn:name rdfs:range rdfs:Literal)) + (triples (string->identifier "platform" (field GeneChip Name)) + (set rdf:type 'gn:platform) + (set gn:name (field GeneChip GeneChipName)))) + +(define-dump dump-probeset + (tables (ProbeSet + (left-join ProbeSetXRef "ON ProbeSetXRef.ProbeSetId = ProbeSet.Id") + (left-join ProbeSetFreeze "ON ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id") + (left-join GeneChip "ON GeneChip.Id = ProbeSet.ChipId"))) + (schema-triples + (gn:name rdfs:range rdfs:Literal)) + (triples (ontology 'gn:probeset_ (field ("IFNULL(ProbeSet.Name, ProbeSet.Id)" + name))) + (set gn:probesetOfDataset + (string->identifier + "dataset" + (field ProbeSetFreeze Name))) + (set gn:mean (annotate-field (field ("IFNULL(ProbeSetXRef.mean, '')" mean)) + '^^xsd:double)) + (set gn:se (annotate-field (field ("IFNULL(ProbeSetXRef.se, '')" se)) + '^^xsd:double)) + (set gn:LRS (annotate-field (field ("IFNULL(ProbeSetXRef.LRS, '')" LRS)) + '^^xsd:double)) + (set gn:pValue (annotate-field (field ("IFNULL(ProbeSetXRef.pValue, '')" pValue)) + '^^xsd:double)) + (set gn:additive (annotate-field (field ("IFNULL(ProbeSetXRef.additive, '')" additive)) + '^^xsd:double)) + (set gn:h2 (annotate-field (field ("IFNULL(ProbeSetXRef.h2, '')" h2)) + '^^xsd:float)) + (set gn:locus (field ProbeSetXRef Locus)) + (set gn:chipOf (string->identifier "platform" (field GeneChip Name))) + (set gn:name (field ProbeSet Name)) + (set gn:symbol (field ProbeSet Symbol)) + (set gn:description (field ProbeSet description)) + (set gn:chr (field ProbeSet Chr)) + (set gn:mb (annotate-field (field ("IFNULL(ProbeSet.Mb, '')" Mb)) '^^xsd:double)) + (set gn:chr_2016 (field ProbeSet Chr_2016)) + (set gn:mb_2016 (annotate-field (field ("IFNULL(ProbeSet.Mb_2016, '')" Mb_2016)) '^^xsd:double)) + (set gn:alias (string-trim-both (field ProbeSet alias))) + (set gn:generif (ontology 'generif: (field ProbeSet GeneId))) + (set gn:genbank (ontology 'nuccore: (field ProbeSet GenbankId))) + (set gn:snp (field ("IFNULL(ProbeSet.SNP, '')" SNP))) + (set gn:blatSeq (string-trim-both (field ProbeSet BlatSeq))) + (set gn:targetSeq (field ProbeSet TargetSeq)) + (set gn:unigene (field ProbeSet UniGeneId)) + (set gn:strandProbe (field ProbeSet Strand_Probe)) + (set gn:strandGene (field ProbeSet Strand_Gene)) + (set gn:omim (ontology 'omim: (field ProbeSet OMIM))) + (set gn:comments (sanitize-rdf-string (field ProbeSet comments))) + (set gn:targetRegion (field ProbeSet Probe_set_target_region)) + (set gn:specificity (annotate-field + (field ("IFNULL(ProbeSet.Probe_set_specificity, '')" Probe_set_specificity)) + '^^xsd:double)) + (set gn:blatScore (annotate-field + (field ("IFNULL(ProbeSet.Probe_set_BLAT_score, '')" Probe_set_BLAT_score)) + '^^xsd:double)) + (set gn:blatMbStart (annotate-field + (field ("IFNULL(ProbeSet.Probe_set_Blat_Mb_start, '')" Probe_set_Blat_Mb_start)) + '^^xsd:double)) + (set gn:blatMbend (annotate-field + (field ("IFNULL(ProbeSet.Probe_set_Blat_Mb_end, '')" Probe_set_Blat_Mb_end)) + '^^xsd:double)) + (set gn:blatMbStart2016 + (annotate-field + (field ("IFNULL(ProbeSet.Probe_set_Blat_Mb_start_2016, '')" Probe_set_Blat_Mb_start_2016)) '^^xsd:double)) + (set gn:blatMbend2016 + (annotate-field + (field ("IFNULL(ProbeSet.Probe_set_Blat_Mb_end_2016, '')" Probe_set_Blat_Mb_end_2016)) '^^xsd:double)) + (set gn:strand (field ProbeSet Probe_set_strand)) + (set gn:noteByRW (field ProbeSet Probe_set_Note_by_RW)) + (set gn:flag (field ProbeSet flag)) + (set gn:symbolH (field ProbeSet Symbol_H)) + (set gn:descriptionH (field ProbeSet Description_H)) + (set gn:chromosomeH (field ProbeSet chromosome_H)) + (set gn:mbH (annotate-field (field ProbeSet MB_H) '^^xsd:double)) + (set gn:aliasH (field ProbeSet alias_H)) + (set gn:geneIdH (field ProbeSet GeneId_H)) + (set gn:chrNum (field ("IFNULL(ProbeSet.chr_num, '')" chr_num))) + (set gn:nameNum (field ("IFNULL(ProbeSet.name_num, '')" name_num))) + (set gn:probeTargetDescription (field ProbeSet Probe_Target_Description)) + (set gn:RefSeq_TranscriptId (ontology 'nuccore: (field ProbeSet RefSeq_TranscriptId))) + (set gn:ENSEMBLGeneId (string-trim-both + (field ProbeSet ENSEMBLGeneId))) + (set gn:Chr_mm8 (field ("IFNULL(ProbeSet.Chr_mm8, '')" Chr_mm8))) + (set gn:Mb_mm8 (field ("IFNULL(ProbeSet.Mb_mm8, '')" Mb_mm8))) + (set gn:probeSetBlatMbStart_mm8 + (annotate-field + (field ("IFNULL(ProbeSet.Probe_set_Blat_Mb_start_mm8, '')" Probe_set_Blat_Mb_start_mm8)) + '^^xsd:double)) + (set gn:probeSetBlatMbEnd_mm8 + (annotate-field + (field ("IFNULL(ProbeSet.Probe_set_Blat_Mb_end_mm8, '')" Probe_set_Blat_Mb_end_mm8)) + '^^xsd:double)) + (set gn:homoloGeneID (ontology + 'homologene: + (field ProbeSet HomoloGeneID))) + (set gn:biotype_ENS (field ProbeSet Biotype_ENS)) + (set gn:proteinName (field ProbeSet ProteinName)) + (set gn:uniProtReference (ontology 'uniprot: + (field ProbeSet UniProtID))) + (set gn:flybase_Id (field ProbeSet Flybase_Id)) + (set gn:RGD_ID (field ("IFNULL(ProbeSet.RGD_ID, '')" RGD_ID))) + (set gn:hgnc (ontology + 'hgnc: (field ProbeSet HGNC_ID))) + (set gn:HMDB_ID (field ProbeSet HMDB_ID)) + (set gn:confidence (field ("IFNULL(ProbeSet.Confidence, '')" Confidence))) + (set gn:chebi_ID (ontology + 'chebi: + (field + ("IFNULL(ProbeSet.ChEBI_ID, '')" + ChEBI_ID)))) + (set gn:CASNumber (field ProbeSet CAS_number)) + (set gn:PubChemID (ontology + 'pubchem: + (field + ("IFNULL(ProbeSet.PubChem_ID, '')" + PubChem_ID)))) + (set gn:chemSpiderID (field ("IFNULL(ProbeSet.ChemSpider_ID, '')" ChemSpider_ID))) + (set gn:uniiID (field ProbeSet UNII_ID)) + (set gn:ECNumber (field ProbeSet EC_number)) + (set gn:keggID (ontology 'kegg: + (field ProbeSet KEGG_ID))) + (set gn:molecularWeight (annotate-field (field ProbeSet Molecular_Weight) '^^xsd:double)) + (set gn:nugowikiID (field ("IFNULL(ProbeSet.Nugowiki_ID, '')" Nugowiki_ID))) + (set gn:type (field ProbeSet Type)) + (set gn:tissue (field ProbeSet Tissue)) + (set gn:primaryName (field ProbeSet PrimaryName)) + (set gn:secondaryNames (field ProbeSet SecondaryNames)) + (set gn:peptideSequence (field ProbeSet PeptideSequence)))) + +;; Molecular Traits are also referred to as ProbeSets +(define-dump dump-probesetfreeze + (tables (ProbeSetFreeze + (left-join ProbeFreeze "USING (ProbeFreezeId)") + (left-join AvgMethod "ON AvgMethod.AvgMethodId = ProbeSetFreeze.AvgID") + (left-join InbredSet "ON ProbeFreeze.InbredSetId=InbredSet.Id") + (left-join Tissue "USING (TissueId)")) + "WHERE ProbeSetFreeze.public > 0 GROUP BY ProbeFreeze.Id") + (schema-triples + (gn:molecularTrait rdfs:range rdfs:Literal)) + (triples + (string->identifier "dataset" (field ProbeSetFreeze Name)) + (set rdf:type 'gn:dataset) + (set gn:avgMethod (string->identifier "avgmethod" (field AvgMethod Name))) + (set gn:fullName (field ProbeSetFreeze FullName)) + (set gn:shortName (field ProbeSetFreeze ShortName)) + (set dct:created (annotate-field + (field ProbeSetFreeze CreateTime) + '^^xsd:datetime)) + (set gn:dataScale (field ProbeSetFreeze DataScale)) + (set gn:tissueName (string->identifier "tissue" (field Tissue Short_Name))) + (set gn:datasetOfInbredSet + (string->identifier "inbredSet" (field InbredSet Name InbredSetName))))) + + + +(call-with-target-database + %connection-settings + (lambda (db) + (with-output-to-file (string-append %dump-directory "dump-probeset.ttl") + (lambda () + (prefix "chebi:" "<http://purl.obolibrary.org/obo/CHEBI_>") + (prefix "dct:" "<http://purl.org/dc/terms/>") + (prefix "foaf:" "<http://xmlns.com/foaf/0.1/>") + (prefix "generif:" "<http://www.ncbi.nlm.nih.gov/gene?cmd=Retrieve&dopt=Graphics&list_uids=>") + (prefix "gn:" "<http://genenetwork.org/>") + (prefix "hgnc:" "<http://bio2rdf.org/hgnc:>") + (prefix "homologene:" "<https://bio2rdf.org/homologene:>") + (prefix "kegg:" "<http://bio2rdf.org/ns/kegg#>") + (prefix "molecularTrait:" "<http://genenetwork.org/molecular-trait/>") + (prefix "nuccore:" "<https://www.ncbi.nlm.nih.gov/nuccore/>") + (prefix "omim:" "<https://www.omim.org/entry/>") + (prefix "owl:" "<http://www.w3.org/2002/07/owl#>") + (prefix "phenotype:" "<http://genenetwork.org/phenotype/>") + (prefix "pubchem:" "<https://pubchem.ncbi.nlm.nih.gov/>") + (prefix "pubmed:" "<http://rdf.ncbi.nlm.nih.gov/pubmed/>") + (prefix "rdf:" "<http://www.w3.org/1999/02/22-rdf-syntax-ns#>") + (prefix "rdfs:" "<http://www.w3.org/2000/01/rdf-schema#>") + (prefix "taxon:" "<http://purl.uniprot.org/taxonomy/>") + (prefix "uniprot:" "<http://purl.uniprot.org/uniprot/>") + (prefix "up:" "<http://purl.uniprot.org/core/>") + (prefix "xsd:" "<http://www.w3.org/2001/XMLSchema#>") + (newline) + (dump-gene-chip db) + (dump-probeset db) + (dump-probesetfreeze db)) + #:encoding "utf8"))) diff --git a/examples/dump-publication.scm b/examples/dump-publication.scm new file mode 100755 index 0000000..bed957a --- /dev/null +++ b/examples/dump-publication.scm @@ -0,0 +1,90 @@ +#! /usr/bin/env guile +!# + +(use-modules (srfi srfi-1) + (srfi srfi-26) + (ice-9 match) + (ice-9 regex) + (dump strings) + (dump sql) + (dump triples) + (dump special-forms)) + + + +(define %connection-settings + (call-with-input-file (list-ref (command-line) 1) + read)) + +(define %dump-directory + (list-ref (command-line) 2)) + + + +(define-dump dump-publication + (tables (Publication)) + (schema-triples + (gn:pubMedId rdfs:range rdfs:Literal) + (gn:title rdfs:range rdfs:Literal) + (gn:journal rdfs:range rdfs:Literal) + (gn:volume rdfs:range rdfs:Literal) + (gn:pages rdfs:range rdfs:Literal) + (gn:month rdfs:range rdfs:Literal) + (gn:year rdfs:range rdfs:Literal) + (gn:author rdfs:range rdfs:Literal) + (gn:abstract rdfs:range rdfs:Literal)) + (triples + (let ((pmid (field + ("IF(Publication.PubMed_ID IS NULL, '', CONVERT(Publication.PubMed_Id, INT))" + pmid))) + (publication-id (field Publication Id))) + (if (string-null? pmid) + (string->identifier "publication" + (number->string publication-id)) + (ontology 'pubmed: pmid))) + (set rdf:type 'gn:publication) + (set gn:pubMedId (field ("IFNULL(PubMed_ID, '')" pubmedId))) + (set gn:title (field Publication Title)) + (set gn:journal (field Publication Journal)) + (set gn:volume (field Publication Volume)) + (set gn:pages (field Publication Pages)) + (set gn:month (field Publication Month)) + (set gn:year (field Publication Year)) + (multiset gn:author + ;; The authors field is a comma + ;; separated list. Split it. + (map string-trim (string-split (sanitize-rdf-string (field Publication Authors)) #\,))) + (set gn:abstract + (sanitize-rdf-string (field Publication Abstract))))) + + + +(call-with-target-database + %connection-settings + (lambda (db) + (with-output-to-file (string-append %dump-directory "dump-publication.ttl") + (lambda () + (prefix "chebi:" "<http://purl.obolibrary.org/obo/CHEBI_>") + (prefix "dct:" "<http://purl.org/dc/terms/>") + (prefix "foaf:" "<http://xmlns.com/foaf/0.1/>") + (prefix "generif:" "<http://www.ncbi.nlm.nih.gov/gene?cmd=Retrieve&dopt=Graphics&list_uids=>") + (prefix "gn:" "<http://genenetwork.org/>") + (prefix "hgnc:" "<http://bio2rdf.org/hgnc:>") + (prefix "homologene:" "<https://bio2rdf.org/homologene:>") + (prefix "kegg:" "<http://bio2rdf.org/ns/kegg#>") + (prefix "molecularTrait:" "<http://genenetwork.org/molecular-trait/>") + (prefix "nuccore:" "<https://www.ncbi.nlm.nih.gov/nuccore/>") + (prefix "omim:" "<https://www.omim.org/entry/>") + (prefix "owl:" "<http://www.w3.org/2002/07/owl#>") + (prefix "phenotype:" "<http://genenetwork.org/phenotype/>") + (prefix "pubchem:" "<https://pubchem.ncbi.nlm.nih.gov/>") + (prefix "pubmed:" "<http://rdf.ncbi.nlm.nih.gov/pubmed/>") + (prefix "rdf:" "<http://www.w3.org/1999/02/22-rdf-syntax-ns#>") + (prefix "rdfs:" "<http://www.w3.org/2000/01/rdf-schema#>") + (prefix "taxon:" "<http://purl.uniprot.org/taxonomy/>") + (prefix "uniprot:" "<http://purl.uniprot.org/uniprot/>") + (prefix "up:" "<http://purl.uniprot.org/core/>") + (prefix "xsd:" "<http://www.w3.org/2001/XMLSchema#>") + (newline) + (dump-publication db)) + #:encoding "utf8"))) diff --git a/examples/dump-species-metadata.scm b/examples/dump-species-metadata.scm new file mode 100755 index 0000000..ed4113c --- /dev/null +++ b/examples/dump-species-metadata.scm @@ -0,0 +1,141 @@ +#! /usr/bin/env guile +!# + +(use-modules (srfi srfi-1) + (srfi srfi-26) + (ice-9 match) + (ice-9 regex) + (dump strings) + (dump sql) + (dump triples) + (dump special-forms)) + + + +(define %connection-settings + (call-with-input-file (list-ref (command-line) 1) + read)) + +(define %dump-directory + (list-ref (command-line) 2)) + + + +(define-dump dump-species + (tables (Species)) + (schema-triples + (gn:name rdfs:range rdfs:Literal) + (gn:displayName rdfs:range rdfs:Literal) + (gn:binomialName rdfs:range rdfs:Literal) + (gn:family rdfs:range rdfs:Literal)) + (triples (string->identifier "species" (field Species FullName)) + (set rdf:type 'gn:species) + (set gn:name (field Species SpeciesName)) + (set gn:displayName (field Species MenuName)) + (set gn:binomialName (field Species FullName)) + (set gn:family (field Species Family)) + (set up:organism (ontology 'taxon: (field Species TaxonomyId))))) + +(define-dump dump-strain + (tables (Strain + (join Species "ON Strain.SpeciesId = Species.SpeciesId"))) + (schema-triples + (gn:strainOfSpecies rdfs:domain gn:strain) + (gn:strainOfSpecies rdfs:range gn:species) + (gn:name rdfs:range rdfs:Literal) + (gn:alias rdfs:range rdfs:Literal) + (gn:symbol rdfs:range rdfs:Literal)) + (triples (string->identifier + "strain" + (regexp-substitute/global + #f "[^A-Za-z0-9:]" + (field ("CAST(CONVERT(BINARY CONVERT(Strain.Name USING latin1) USING utf8) AS VARCHAR(15000))" StrainName)) + 'pre "_" 'post)) + (set rdf:type 'gn:strain) + (set gn:strainOfSpecies + (string->identifier "species" (field Species FullName))) + ;; Name, and maybe a second name + (set gn:name (sanitize-rdf-string (field Strain Name))) + (set gn:name (sanitize-rdf-string (field Strain Name2))) + (set gn:alias (sanitize-rdf-string (field Strain Alias))) + (set gn:symbol (field Strain Symbol)))) + +(define-dump dump-mapping-method + (tables (MappingMethod)) + (triples (string->identifier "mappingMethod" (field MappingMethod Name)) + (set rdf:type 'gn:mappingMethod))) + +(define-dump dump-inbred-set + (tables (InbredSet + (left-join Species "ON InbredSet.SpeciesId=Species.Id") + (left-join MappingMethod + "ON InbredSet.MappingMethodId=MappingMethod.Id"))) + (schema-triples + (gn:fullName rdfs:range rdfs:Literal) + (gn:geneticType rdfs:range rdfs:Literal) + (gn:inbredSetCode rdfs:range rdfs:Literal) + (gn:inbredFamily rdfs:range rdfs:Literal) + (gn:inbredSetOfSpecies rdfs:range gn:species) + (gn:inbredSetType rdfs:range rdfs:Literal) + (gn:phenotype rdfs:range gn:inbredSetType) + (gn:genotype rdfs:range gn:inbredSetType) + (gn:inbredSetOfMappingMethod rdfs:range gn:mappingMethod)) + (triples (string->identifier "inbredSet" (field InbredSet Name)) + (set rdf:type 'gn:inbredSet) + (set gn:binomialName (field InbredSet FullName)) + (set gn:geneticType (field InbredSet GeneticType)) + (set gn:inbredFamily (field InbredSet Family)) + (set gn:inbredSetOfMappingMethod (field MappingMethod Name)) + (set gn:inbredSetCode (field InbredSet InbredSetCode)) + (set gn:inbredSetOfSpecies + (string->identifier "species" (field Species FullName BinomialName))) + (set gn:genotype + (field ("IF ((SELECT PublishFreeze.Name FROM PublishFreeze WHERE PublishFreeze.InbredSetId = InbredSet.Id LIMIT 1) IS NOT NULL, 'Traits and Cofactors', '')" genotypeP))) + (set gn:phenotype + (field ("IF ((SELECT GenoFreeze.Name FROM GenoFreeze WHERE GenoFreeze.InbredSetId = InbredSet.Id LIMIT 1) IS NOT NULL, 'DNA Markers and SNPs', '')" phenotypeP))))) + +(define-dump dump-avg-method + ;; The Name and Normalization fields seem to be the same. Dump only + ;; the Name field. + (tables (AvgMethod)) + (schema-triples + (gn:name rdfs:range rdfs:Literal)) + (triples (string->identifier "avgmethod" (field AvgMethod Name)) + (set rdf:type 'gn:avgMethod) + (set gn:name (field AvgMethod Name)))) + + + +(call-with-target-database + %connection-settings + (lambda (db) + (with-output-to-file (string-append %dump-directory "dump-species-metadata.ttl") + (lambda () + (prefix "chebi:" "<http://purl.obolibrary.org/obo/CHEBI_>") + (prefix "dct:" "<http://purl.org/dc/terms/>") + (prefix "foaf:" "<http://xmlns.com/foaf/0.1/>") + (prefix "generif:" "<http://www.ncbi.nlm.nih.gov/gene?cmd=Retrieve&dopt=Graphics&list_uids=>") + (prefix "gn:" "<http://genenetwork.org/>") + (prefix "hgnc:" "<http://bio2rdf.org/hgnc:>") + (prefix "homologene:" "<https://bio2rdf.org/homologene:>") + (prefix "kegg:" "<http://bio2rdf.org/ns/kegg#>") + (prefix "molecularTrait:" "<http://genenetwork.org/molecular-trait/>") + (prefix "nuccore:" "<https://www.ncbi.nlm.nih.gov/nuccore/>") + (prefix "omim:" "<https://www.omim.org/entry/>") + (prefix "owl:" "<http://www.w3.org/2002/07/owl#>") + (prefix "phenotype:" "<http://genenetwork.org/phenotype/>") + (prefix "pubchem:" "<https://pubchem.ncbi.nlm.nih.gov/>") + (prefix "pubmed:" "<http://rdf.ncbi.nlm.nih.gov/pubmed/>") + (prefix "rdf:" "<http://www.w3.org/1999/02/22-rdf-syntax-ns#>") + (prefix "rdfs:" "<http://www.w3.org/2000/01/rdf-schema#>") + (prefix "taxon:" "<http://purl.uniprot.org/taxonomy/>") + (prefix "uniprot:" "<http://purl.uniprot.org/uniprot/>") + (prefix "up:" "<http://purl.uniprot.org/core/>") + (prefix "xsd:" "<http://www.w3.org/2001/XMLSchema#>") + (newline) + (dump-species db) + (dump-strain db) + (dump-mapping-method db) + (dump-inbred-set db) + (dump-avg-method db)) + #:encoding "utf8"))) diff --git a/examples/dump-tissue.scm b/examples/dump-tissue.scm new file mode 100755 index 0000000..3d55383 --- /dev/null +++ b/examples/dump-tissue.scm @@ -0,0 +1,66 @@ +#! /usr/bin/env guile +!# + +(use-modules (srfi srfi-1) + (srfi srfi-26) + (ice-9 match) + (ice-9 regex) + (dump strings) + (dump sql) + (dump triples) + (dump special-forms)) + + + +(define %connection-settings + (call-with-input-file (list-ref (command-line) 1) + read)) + +(define %dump-directory + (list-ref (command-line) 2)) + + + +(define-dump dump-tissue + ;; The Name and TissueName fields seem to be identical. BIRN_lex_ID + ;; and BIRN_lex_Name are mostly NULL. + (tables (Tissue)) + (schema-triples + (gn:name rdfs:range rdfs:Literal)) + ;; Hopefully the Short_Name field is distinct and can be used as an + ;; identifier. + (triples (string->identifier "tissue" (field Tissue Short_Name)) + (set rdf:type 'gn:tissue) + (set gn:name (field Tissue Name)))) + + + +(call-with-target-database + %connection-settings + (lambda (db) + (with-output-to-file (string-append %dump-directory "dump-tissue.ttl") + (lambda () + (prefix "chebi:" "<http://purl.obolibrary.org/obo/CHEBI_>") + (prefix "dct:" "<http://purl.org/dc/terms/>") + (prefix "foaf:" "<http://xmlns.com/foaf/0.1/>") + (prefix "generif:" "<http://www.ncbi.nlm.nih.gov/gene?cmd=Retrieve&dopt=Graphics&list_uids=>") + (prefix "gn:" "<http://genenetwork.org/>") + (prefix "hgnc:" "<http://bio2rdf.org/hgnc:>") + (prefix "homologene:" "<https://bio2rdf.org/homologene:>") + (prefix "kegg:" "<http://bio2rdf.org/ns/kegg#>") + (prefix "molecularTrait:" "<http://genenetwork.org/molecular-trait/>") + (prefix "nuccore:" "<https://www.ncbi.nlm.nih.gov/nuccore/>") + (prefix "omim:" "<https://www.omim.org/entry/>") + (prefix "owl:" "<http://www.w3.org/2002/07/owl#>") + (prefix "phenotype:" "<http://genenetwork.org/phenotype/>") + (prefix "pubchem:" "<https://pubchem.ncbi.nlm.nih.gov/>") + (prefix "pubmed:" "<http://rdf.ncbi.nlm.nih.gov/pubmed/>") + (prefix "rdf:" "<http://www.w3.org/1999/02/22-rdf-syntax-ns#>") + (prefix "rdfs:" "<http://www.w3.org/2000/01/rdf-schema#>") + (prefix "taxon:" "<http://purl.uniprot.org/taxonomy/>") + (prefix "uniprot:" "<http://purl.uniprot.org/uniprot/>") + (prefix "up:" "<http://purl.uniprot.org/core/>") + (prefix "xsd:" "<http://www.w3.org/2001/XMLSchema#>") + (newline) + (dump-tissue db)) + #:encoding "utf8"))) |