You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

563 lines
18 KiB

  1. /*
  2. This file is part of BioD.
  3. Copyright (C) 2012 Artem Tarasov <lomereiter@gmail.com>
  4. Permission is hereby granted, free of charge, to any person obtaining a
  5. copy of this software and associated documentation files (the "Software"),
  6. to deal in the Software without restriction, including without limitation
  7. the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. and/or sell copies of the Software, and to permit persons to whom the
  9. Software is furnished to do so, subject to the following conditions:
  10. The above copyright notice and this permission notice shall be included in
  11. all copies or substantial portions of the Software.
  12. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  13. IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  14. FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  15. AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  16. LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  17. FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  18. DEALINGS IN THE SOFTWARE.
  19. */
  20. module bio.bam.validation.alignment;
  21. public import bio.bam.read;
  22. public import bio.bam.tagvalue;
  23. import bio.core.utils.algo;
  24. import std.algorithm;
  25. import std.ascii;
  26. import std.conv;
  27. import std.typetuple;
  28. /**
  29. Alignment validation error types.
  30. InvalidCigar error is accompanied by some CigarError,
  31. InvalidTags is accompanied by some TagError.
  32. */
  33. enum AlignmentError {
  34. EmptyReadName, ///
  35. TooLongReadName, ///
  36. ReadNameContainsInvalidCharacters, ///
  37. PositionIsOutOfRange, ///
  38. QualityDataContainsInvalidElements, ///
  39. InvalidCigar, ///
  40. InvalidTags, ///
  41. DuplicateTagKeys ///
  42. }
  43. /// CIGAR string validation error types.
  44. enum CigarError {
  45. InternalHardClipping, ///
  46. InternalSoftClipping, ///
  47. InconsistentLength ///
  48. }
  49. /// Auxiliary data validation error types.
  50. ///
  51. /// Refers to an individual tag.
  52. enum TagError {
  53. EmptyString, ///
  54. EmptyHexadecimalString, ///
  55. NonPrintableString, ///
  56. NonPrintableCharacter, ///
  57. InvalidHexadecimalString, ///
  58. ExpectedIntegerValue, ///
  59. ExpectedStringValue, ///
  60. InvalidValueType, ///
  61. InvalidQualityString, ///
  62. ExpectedStringWithSameLengthAsSequence ///
  63. }
  64. /// Designates pair of predefined key from SAM/BAM specification
  65. /// and expected type of tags with that key.
  66. struct TagType(string key, T) {
  67. enum Key = key;
  68. alias T Type;
  69. }
  70. /// Compile-time available information about predefined tags
  71. alias TypeTuple!(TagType!("AM", int),
  72. TagType!("AS", int),
  73. TagType!("BC", string),
  74. TagType!("BQ", string),
  75. TagType!("CC", string),
  76. TagType!("CM", int),
  77. TagType!("CP", int),
  78. TagType!("CQ", string),
  79. TagType!("CS", string),
  80. TagType!("E2", string),
  81. TagType!("FI", int),
  82. TagType!("FS", string),
  83. TagType!("FZ", ushort[]),
  84. TagType!("LB", string),
  85. TagType!("H0", int),
  86. TagType!("H1", int),
  87. TagType!("H2", int),
  88. TagType!("HI", int),
  89. TagType!("IH", int),
  90. TagType!("MD", string),
  91. TagType!("MQ", int),
  92. TagType!("NH", int),
  93. TagType!("NM", int),
  94. TagType!("OQ", string),
  95. TagType!("OP", int),
  96. TagType!("OC", string),
  97. TagType!("PG", string),
  98. TagType!("PQ", int),
  99. TagType!("PU", string),
  100. TagType!("Q2", string),
  101. TagType!("R2", string),
  102. TagType!("RG", string),
  103. TagType!("SM", int),
  104. TagType!("TC", int),
  105. TagType!("U2", string),
  106. TagType!("UQ", int))
  107. PredefinedTags;
  108. private template GetKey(U) {
  109. enum GetKey = U.Key;
  110. }
  111. private template PredefinedTagTypeHelper(string s) {
  112. alias PredefinedTags[staticIndexOf!(s, staticMap!(GetKey, PredefinedTags))] PredefinedTagTypeHelper;
  113. }
  114. /// Get predefined tag type by its key, in compile-time.
  115. template PredefinedTagType(string s) {
  116. alias PredefinedTagTypeHelper!(s).Type PredefinedTagType;
  117. }
  118. /**
  119. Abstract class encapsulating visitation of SAM header elements.
  120. */
  121. abstract class AbstractAlignmentValidator {
  122. /// Start validation process.
  123. ///
  124. /// Passing by reference is not only for doing less copying,
  125. /// one might want to attempt to fix invalid fields
  126. /// in onError() methods.
  127. void validate(ref BamRead alignment) {
  128. _visitAlignment(alignment);
  129. }
  130. /** Implement those methods to define your own behaviour.
  131. During validation process, in case of an error the corresponding
  132. method gets called, and is provided the object where the error occurred,
  133. and type of the error. Objects are passed by reference so that they
  134. can be changed (fixed / cleaned up / etc.)
  135. If onError() returns true, that means to continue validation process
  136. for this particular entity. Otherwise, all other validation checks are
  137. skipped and next entity is processed.
  138. */
  139. abstract bool onError(ref BamRead al, AlignmentError error);
  140. abstract bool onError(ref BamRead al, CigarError error); /// ditto
  141. abstract bool onError(string key, const ref Value value, TagError error); /// ditto
  142. private:
  143. // Method names are a bit misleading,
  144. // their return value is NOT whether a field is invalid or not
  145. // but rather whether onError() handlers decide to stop validation
  146. // when the field is invalid.
  147. bool invalidReadName(ref BamRead al) {
  148. // Read name (a.k.a. QNAME) must =~ /^[!-?A-~]{1,255}$/
  149. // according to specification.
  150. if (al.name.length == 0) {
  151. if (!onError(al, AlignmentError.EmptyReadName)) return true;
  152. } else if (al.name.length > 255) {
  153. if (!onError(al, AlignmentError.TooLongReadName)) return true;
  154. } else {
  155. foreach (char c; al.name)
  156. {
  157. if ((c < '!') || (c > '~') || (c == '@')) {
  158. if (!onError(al, AlignmentError.ReadNameContainsInvalidCharacters)) {
  159. return true;
  160. } else {
  161. break;
  162. }
  163. }
  164. }
  165. }
  166. return false;
  167. }
  168. bool invalidPosition(ref BamRead al) {
  169. /// Check that position is in range [-1 .. 2^29 - 2]
  170. if (al.position < -1 || al.position > ((1<<29) - 2)) {
  171. if (!onError(al, AlignmentError.PositionIsOutOfRange)) {
  172. return true;
  173. }
  174. }
  175. return false;
  176. }
  177. bool invalidQualityData(ref BamRead al) {
  178. /// Check quality data
  179. if (!all!"a == 0xFF"(al.base_qualities) &&
  180. !all!"0 <= a && a <= 93"(al.base_qualities))
  181. {
  182. if (!onError(al, AlignmentError.QualityDataContainsInvalidElements)) {
  183. return true;
  184. }
  185. }
  186. return false;
  187. }
  188. static bool internalHardClipping(ref BamRead al) {
  189. return (al.cigar.length > 2 &&
  190. any!"a.type == 'H'"(al.cigar[1..$-1]));
  191. }
  192. static bool internalSoftClipping(ref BamRead al) {
  193. if (al.cigar.length <= 2) return false;
  194. auto cigar = al.cigar;
  195. /// strip H operations from ends
  196. if (cigar[0].type == 'H') {
  197. cigar = cigar[1..$];
  198. }
  199. if (cigar[$-1].type == 'H') {
  200. cigar = cigar[0..$-1];
  201. }
  202. /// check that S operations are at the ends only
  203. return (cigar.length > 2 &&
  204. any!"a.type == 'S'"(cigar[1..$-1]));
  205. }
  206. // Sum of M/I/S/=/X operations must be equal to the sequence length
  207. // if both sequence and CIGAR string are presented.
  208. static bool inconsistentLength(ref BamRead al) {
  209. return (al.sequence_length > 0 &&
  210. al.sequence_length != reduce!`a + b`(0,
  211. map!`a.length`(
  212. filter!`canFind("MIS=X", a.type)`(
  213. al.cigar))));
  214. }
  215. bool invalidCigar(ref BamRead al) {
  216. if (al.cigar.length == 0) return false;
  217. static string check(string s) {
  218. import std.ascii : toUpper;
  219. return (`if (`~s.dup~`(al)`~
  220. ` && !onError(al, CigarError.`~(cast(char)(s[0]-32))~s[1..$]~`)`~
  221. ` && (called_on_error || onError(al, AlignmentError.InvalidCigar)))`~
  222. `{`~
  223. ` return true;`~
  224. `}`).idup;
  225. }
  226. bool called_on_error = false;
  227. mixin(check("internalHardClipping"));
  228. mixin(check("internalSoftClipping"));
  229. mixin(check("inconsistentLength"));
  230. return false;
  231. }
  232. // Check tags, a lot of them are predefined in the specification
  233. // and have to satisfy certain requirements.
  234. bool invalidTags(ref BamRead al) {
  235. bool all_tags_are_good = true;
  236. void someTagIsBad() {
  237. if (all_tags_are_good) {
  238. if (!onError(al, AlignmentError.InvalidTags)) return;
  239. }
  240. all_tags_are_good = false;
  241. }
  242. /// Check that all tag keys are distinct.
  243. bool all_distinct = true;
  244. // Optimize for small number of tags
  245. ushort[256] keys = void;
  246. size_t i = 0;
  247. // Check each tag in turn.
  248. foreach (k, v; al) {
  249. if (!isValid(k, v, al)) {
  250. someTagIsBad();
  251. }
  252. if (i < keys.length) {
  253. keys[i] = *cast(ushort*)(k.ptr);
  254. if (all_distinct) {
  255. for (size_t j = 0; j < i; ++j) {
  256. if (keys[i] == keys[j]) {
  257. all_distinct = false;
  258. break;
  259. }
  260. }
  261. }
  262. i += 1;
  263. } else {
  264. if (all_distinct) {
  265. // must be exactly one
  266. int found = 0;
  267. foreach (k2, v2; al) {
  268. if (*cast(ushort*)(k2.ptr) == *cast(ushort*)(k.ptr)) {
  269. if (found == 1) {
  270. all_distinct = false;
  271. break;
  272. } else {
  273. ++found;
  274. }
  275. }
  276. }
  277. }
  278. }
  279. }
  280. if (!all_distinct) {
  281. if (!onError(al, AlignmentError.DuplicateTagKeys)) return true;
  282. }
  283. return false;
  284. }
  285. void _visitAlignment(ref BamRead al) {
  286. if (invalidReadName(al)) return;
  287. if (invalidPosition(al)) return;
  288. if (invalidQualityData(al)) return;
  289. if (invalidCigar(al)) return;
  290. if (invalidTags(al)) return;
  291. }
  292. bool isValid(string key, const ref Value value, const ref BamRead al) {
  293. bool result = true;
  294. if (value.is_hexadecimal_string()) {
  295. auto str = cast(string)value;
  296. if (str.length == 0) {
  297. if (!onError(key, value, TagError.EmptyHexadecimalString)) {
  298. return false;
  299. }
  300. result = false;
  301. }
  302. /// check that it contains only 0..9a-fA-F characters
  303. if (!all!(isHexDigit)(str)) {
  304. if (!onError(key, value, TagError.InvalidHexadecimalString)) {
  305. return false;
  306. }
  307. result = false;
  308. }
  309. } else if (value.is_character()) {
  310. /// character must be [!-~]
  311. auto c = cast(char)value;
  312. if (!(c >= '!' && c <= '~')) {
  313. if (!onError(key, value, TagError.NonPrintableCharacter)) {
  314. return false;
  315. }
  316. result = false;
  317. }
  318. } else if (value.is_string()) {
  319. auto str = cast(string)value;
  320. if (str.length == 0) {
  321. if (!onError(key, value, TagError.EmptyString)) {
  322. return false;
  323. }
  324. result = false;
  325. }
  326. /// string must be [ !-~]+
  327. if (!all!"a >= ' ' && a <= '~'"(str)) {
  328. if (!onError(key, value, TagError.NonPrintableString)) {
  329. return false;
  330. }
  331. result = false;
  332. }
  333. }
  334. /// check various tags from SAM/BAM specification
  335. if (!additionalChecksIfTheTagIsPredefined(key, value, al)) {
  336. result = false;
  337. }
  338. return result;
  339. }
  340. // There're some requirements for predefined tags to be checked
  341. // such as type, length in some cases, or even some regular expression.
  342. // See page 6 of SAM/BAM specification.
  343. bool additionalChecksIfTheTagIsPredefined(string key, const ref Value value,
  344. const ref BamRead al)
  345. {
  346. bool result = true;
  347. // Creates a switch for all predefined tag keys.
  348. string switchTagKey() {
  349. char[] cs;
  350. foreach (t; PredefinedTags) {
  351. cs ~= `case "`~t.Key~`":`~
  352. ` if (!checkTagValue!"`~t.Key~`"(value, al)) {`~
  353. ` result = false;`~
  354. ` }`~
  355. ` break;`.dup;
  356. }
  357. return "switch (key) { " ~ cs.idup ~ " default : break; }";
  358. }
  359. mixin(switchTagKey());
  360. return result;
  361. }
  362. // Supposed to be inlined in the above switch
  363. bool checkTagValue(string s)(const ref Value value, const ref BamRead al) {
  364. bool result = true;
  365. /// 1. Check type.
  366. static if (is(PredefinedTagType!s == int)) {
  367. if (!value.is_integer) {
  368. if (!onError(s, value, TagError.ExpectedIntegerValue)) {
  369. return false;
  370. }
  371. result = false;
  372. }
  373. } else if (is(PredefinedTagType!s == string)) {
  374. // Notice that there are no 'H'-typed predefined tags,
  375. // and they are almost unused and therefore are not likely
  376. // to appear in the future.
  377. if (!value.is_string || value.bam_typeid == 'H') {
  378. if (!onError(s, value, TagError.ExpectedStringValue)) {
  379. return false;
  380. }
  381. result = false;
  382. }
  383. } else {
  384. if (value.tag != GetTypeId!(PredefinedTagType!s)) {
  385. if (!onError(s, value, TagError.InvalidValueType)) {
  386. return false;
  387. }
  388. result = false;
  389. }
  390. }
  391. /// 2. For tags which contain quality as a string,
  392. /// check that all characters are valid
  393. static if (staticIndexOf!(s, "CQ", "E2", "OQ", "Q2", "U2") != -1) {
  394. auto str = cast(string)value;
  395. if (str != "*" && !all!"a >= '!' && a <= '~'"(str)) {
  396. if (!onError(s, value, TagError.InvalidQualityString)) {
  397. return false;
  398. }
  399. result = false;
  400. }
  401. }
  402. /// 3. In a couple of cases values are required to be
  403. /// of the same length as the read sequence.
  404. static if (staticIndexOf!(s, "BQ", "E2") != -1) {
  405. if ((cast(string)value).length != al.sequence_length) {
  406. if (!onError(s, value, TagError.ExpectedStringWithSameLengthAsSequence)) {
  407. return false;
  408. }
  409. }
  410. }
  411. /// 4. MD tag ought to: a) match /^[0-9]+(([A-Z]|\^[A-Z]+)[0-9]+)*$/
  412. /// b) match CIGAR string (TODO?)
  413. static if (s == "MD") {
  414. /// a) check regular expression
  415. auto s = cast(string)value;
  416. bool valid = true;
  417. if (s.length == 0) valid = false;
  418. if (!isDigit(s[0])) valid = false;
  419. size_t i = 1;
  420. while (i < s.length && isDigit(s[i]))
  421. ++i;
  422. while (i < s.length) {
  423. if (isUpper(s[i])) {
  424. ++i; // [A-Z]
  425. } else if (s[i] == '^') { // ^[A-Z]+
  426. ++i;
  427. if (i == s.length || !isUpper(s[i])) {
  428. valid = false;
  429. break;
  430. }
  431. while (i < s.length && isUpper(s[i]))
  432. ++i;
  433. } else {
  434. valid = false;
  435. break;
  436. }
  437. // now [0-9]+
  438. if (i == s.length || !isDigit(s[i])) {
  439. valid = false;
  440. break;
  441. }
  442. while (i < s.length && isDigit(s[i]))
  443. ++i;
  444. }
  445. if (i < s.length) {
  446. valid = false;
  447. }
  448. if (!valid) result = false;
  449. }
  450. return result;
  451. }
  452. }
  453. final private class BooleanValidator : AbstractAlignmentValidator {
  454. bool result;
  455. override void validate(ref BamRead al) {
  456. result = true;
  457. super.validate(al);
  458. }
  459. override bool onError(ref BamRead al, AlignmentError e) {
  460. return (result = false);
  461. }
  462. override bool onError(ref BamRead al, CigarError e) {
  463. return (result = false);
  464. }
  465. override bool onError(string key, const ref Value val, TagError e) {
  466. return (result = false);
  467. }
  468. }
  469. private static BooleanValidator booleanValidator;
  470. static this() {
  471. booleanValidator = new BooleanValidator();
  472. }
  473. /// Check if alignment is valid
  474. bool isValid(BamRead alignment) {
  475. booleanValidator.validate(alignment);
  476. return booleanValidator.result;
  477. }