about summary refs log tree commit diff
path: root/src/routes/genoupload.rs
blob: bd12018e44d77b3b79b84eaa460f6b201dbaff47 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
//! `POST /uploads/genotypes`: upload a genotype matrix that *updates* an
//! existing population.
//!
//! Tab-delimited, header = individual names from column 4, column 1 the marker,
//! 2 the chromosome, 3 the position in Mb. Values must be the same kind as the
//! base population's (letters, whole numbers or probabilities) -- see
//! `src/genoupload.rs` for what is enforced and why.
//!
//! This is an update path, not an ingest: at most 50 individuals the base does
//! not have, at least one that it does, and at most 200 000 markers. Anything
//! larger is the sysadmin's to organise.

use axum::Json;
use axum::extract::{Path, Query};
use axum::http::StatusCode;
use serde::Deserialize;

use gnauth_macros::gnauth;

use crate::auth::AuthToken;
use crate::baskets::BasketError;
use crate::config::Config;
use crate::error::{ApiError, Unauthorized};
use crate::genotypes;
use crate::genoupload::{self, GenotypeInfo};
use crate::response::{Ctx, Envelope, RequestInfo};

#[derive(Deserialize, utoipa::ToSchema)]
pub struct UploadGenotype {
    /// `<user>-<GENONAME>`: the name this upload is known by.
    pub name: String,
    /// The population it updates, e.g. "BXD.4".
    pub base_version: String,
    /// What changed, for the listing.
    pub description: Option<String>,
    /// The matrix: tab-delimited, one header row, then one row per marker.
    pub contents: String,
}

#[derive(Deserialize, utoipa::IntoParams)]
pub struct MarkerPage {
    /// One-based page. Default 1.
    pub page: Option<u32>,
    /// Markers per page. Default 100, at most 10000.
    pub limit: Option<u32>,
}

fn map_error(error: BasketError, rid: RequestInfo, operation: &'static str) -> ApiError {
    match error {
        BasketError::NotFound(message) => ApiError::new(
            StatusCode::NOT_FOUND,
            "genotype_upload_not_found",
            "Uploaded genotype not found",
            message,
            operation,
        )
        .with_request_id(rid),
        BasketError::Invalid(message) => ApiError::new(
            StatusCode::UNPROCESSABLE_ENTITY,
            "invalid_genotype_upload",
            "Invalid genotype upload",
            message,
            operation,
        )
        .with_request_id(rid),
        BasketError::Store(message) => ApiError::new(
            StatusCode::INTERNAL_SERVER_ERROR,
            "genotype_upload_store_error",
            "The upload store failed",
            message,
            operation,
        )
        .with_request_id(rid),
    }
}

#[utoipa::path(post, path = "/uploads/genotypes",
    summary = "Upload a genotype matrix that updates an existing population",
    description = "Tab-delimited: the header names the individuals from column 4 on, column 1 \
is the marker, 2 the chromosome, 3 the position in Mb. The base population must exist; at most 50 \
individuals may be new and at least one must overlap, and at most 200000 markers -- anything \
larger the sysadmin has to organise. Values must be the same kind as the base's: single letters \
(A/B/H), whole numbers (0/1/2) or probabilities (0.02).",
    request_body = UploadGenotype,
    responses(
        (status = 201, description = "The stored upload", body = Envelope<GenotypeInfo>),
        Unauthorized,
        (status = 404, description = "No such base population", body = ApiError),
        (status = 422, description = "The matrix is malformed or does not fit the base", body = ApiError)
    )
)]
#[gnauth(token_only)]
pub async fn upload_genotype(
    ctx: Ctx,
    token: AuthToken,
    Json(request): Json<UploadGenotype>,
) -> Result<(StatusCode, Json<Envelope<GenotypeInfo>>), ApiError> {
    let rid = ctx.request_info();
    let config = Config::load();
    let owner = token.sub.clone();
    let dir = config.basket_dir.clone();
    let root = config.genotype_files_dir.clone();
    let base_version = request.base_version.clone();

    let parsed = genoupload::parse(&request.contents)
        .map_err(|error| map_error(error, rid.clone(), "upload_genotype"))?;

    // The population being updated: its individuals and the kind of value its
    // matrix holds.
    let base_for_job = base_version.clone();
    let (base_kind, base_individuals) = tokio::task::spawn_blocking(move || {
        let path = genotypes::resolve_named(std::path::Path::new(&root), &base_for_job)
            .map_err(|error| BasketError::NotFound(error.to_string()))?;
        genoupload::base_shape(&path)
    })
    .await
    .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone(), "upload_genotype"))?
    .map_err(|error| map_error(error, rid.clone(), "upload_genotype"))?;
    let (overlap, new_individuals) =
        genoupload::check_base(&parsed, &base_individuals, base_kind)
            .map_err(|error| map_error(error, rid.clone(), "upload_genotype"))?;

    let name = request.name.clone();
    let description = request.description.clone();
    let contents = request.contents.clone();
    // Kept for the note: the parse and the base version move into the closure.
    let base_for_note = base_version.clone();
    let markers_count = parsed.markers.len();
    let kind_label = parsed.kind.as_str();
    let stored = tokio::task::spawn_blocking(move || {
        genoupload::create(
            std::path::Path::new(&dir),
            &owner,
            &name,
            &base_version,
            description.as_deref(),
            &parsed,
            overlap,
            new_individuals,
            &contents,
        )
    })
    .await
    .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone(), "upload_genotype"))?
    .map_err(|error| map_error(error, rid.clone(), "upload_genotype"))?;

    let mut envelope = ctx
        .envelope(stored)
        .with_note(&format!(
            "Updates '{base_for_note}': {overlap} individual(s) shared with the base, \
             {new_individuals} new, {markers_count} marker(s), values as {kind_label}."
        ))
        .with_note(
            "Kept in your own SQLite file. A mapping reads it as a GN .geno: GET \
             /uploads/genotypes/{name}/geno writes it out and gives the path for `genotype_file`.",
        );
    envelope = envelope.with_links(crate::suggest::root_links());
    Ok((StatusCode::CREATED, Json(envelope)))
}

#[utoipa::path(get, path = "/uploads/genotypes",
    summary = "This user's uploaded genotypes, newest first",
    responses(
        (status = 200, description = "Every upload of this user", body = Envelope<Vec<GenotypeInfo>>),
        Unauthorized
    )
)]
#[gnauth(token_only)]
pub async fn list_genotypes(
    ctx: Ctx,
    token: AuthToken,
) -> Result<Json<Envelope<Vec<GenotypeInfo>>>, ApiError> {
    let rid = ctx.request_info();
    let owner = token.sub.clone();
    let dir = Config::load().basket_dir;
    let uploads = tokio::task::spawn_blocking(move || {
        genoupload::list(std::path::Path::new(&dir), &owner)
    })
    .await
    .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone(), "list_genotypes"))?
    .map_err(|error| map_error(error, rid.clone(), "list_genotypes"))?;
    let mut envelope = ctx
        .envelope(uploads)
        .with_note("Uploads are yours, kept in your own SQLite file; they update a population, \
                    they do not replace one.")
        .with_links(crate::suggest::root_links());
    envelope = envelope.with_note("Upload one: POST /uploads/genotypes.");
    Ok(Json(envelope))
}

#[utoipa::path(get, path = "/uploads/genotypes/{name}",
    summary = "One uploaded genotype's metadata",
    params(("name" = String, Path, description = "Upload name, e.g. ``user-GENONAME``")),
    responses(
        (status = 200, description = "The upload", body = Envelope<GenotypeInfo>),
        Unauthorized,
        (status = 404, description = "No such upload for this user", body = ApiError)
    )
)]
#[gnauth(token_only)]
pub async fn get_genotype(
    ctx: Ctx,
    token: AuthToken,
    Path(name): Path<String>,
) -> Result<Json<Envelope<GenotypeInfo>>, ApiError> {
    let rid = ctx.request_info();
    let owner = token.sub.clone();
    let dir = Config::load().basket_dir;
    let wanted = name.clone();
    let upload = tokio::task::spawn_blocking(move || {
        genoupload::get(std::path::Path::new(&dir), &owner, &wanted)
    })
    .await
    .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone(), "get_genotype"))?
    .map_err(|error| map_error(error, rid.clone(), "get_genotype"))?;
    let mut envelope = ctx
        .envelope(upload.clone())
        .with_note("`overlap` and `new_individuals` are measured against the base population.")
        .with_next(
            &format!("/uploads/genotypes/{}/markers", crate::response::path_segment(&name)),
            "The markers in this upload, paged",
        )
        .with_next(
            &format!("/uploads/genotypes/{}/geno", crate::response::path_segment(&name)),
            "Write it out as a GN .geno and get the path for `genotype_file`",
        );
    envelope = envelope.with_links(crate::suggest::root_links());
    Ok(Json(envelope))
}

#[utoipa::path(get, path = "/uploads/genotypes/{name}/markers",
    summary = "The markers in an uploaded genotype, paged",
    params(("name" = String, Path, description = "Upload name"), MarkerPage),
    responses(
        (status = 200, description = "marker, chr, pos rows (meta.columns names them)", body = Envelope<serde_json::Value>),
        Unauthorized,
        (status = 404, description = "No such upload for this user", body = ApiError)
    )
)]
#[gnauth(token_only)]
pub async fn genotype_markers(
    ctx: Ctx,
    token: AuthToken,
    Path(name): Path<String>,
    Query(params): Query<MarkerPage>,
) -> Result<Json<Envelope<serde_json::Value>>, ApiError> {
    let rid = ctx.request_info();
    let owner = token.sub.clone();
    let dir = Config::load().basket_dir;
    let wanted = name.clone();
    let contents = tokio::task::spawn_blocking(move || {
        genoupload::contents(std::path::Path::new(&dir), &owner, &wanted)
    })
    .await
    .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone(), "genotype_markers"))?
    .map_err(|error| map_error(error, rid.clone(), "genotype_markers"))?;
    let parsed = genoupload::parse(&contents)
        .map_err(|error| map_error(error, rid.clone(), "genotype_markers"))?;
    let limit = params.limit.unwrap_or(100).clamp(1, 10_000) as usize;
    let page = params.page.unwrap_or(1).max(1) as usize;
    let rows: Vec<serde_json::Value> = parsed
        .markers
        .iter()
        .skip((page - 1) * limit)
        .take(limit)
        .map(|(marker, chr, pos)| serde_json::json!([marker, chr, pos]))
        .collect();
    let num_rows = rows.len();
    let mut envelope = ctx
        .envelope(serde_json::Value::Array(rows))
        .with_columns(
            ["marker", "chr", "pos"]
                .iter()
                .map(|name| name.to_string())
                .collect(),
            num_rows,
        )
        .with_note(&format!(
            "{} markers in this upload; pos is in Mb, as uploaded.",
            parsed.markers.len()
        ));
    let pages = parsed.markers.len().div_ceil(limit).max(1);
    if page < pages {
        envelope = envelope.with_next(
            &format!(
                "/uploads/genotypes/{}/markers?page={}&limit={limit}",
                crate::response::path_segment(&name),
                page + 1
            ),
            "The next page of markers",
        );
    }
    envelope = envelope.with_next(
        &format!("/uploads/genotypes/{}", crate::response::path_segment(&name)),
        "This upload's metadata",
    );
    Ok(Json(envelope))
}

#[utoipa::path(get, path = "/uploads/genotypes/{name}/geno",
    summary = "Write an uploaded genotype out as a GN .geno",
    description = "Materialises the upload as the layout every R script here reads (Chr, Locus, \
cM, Mb, individuals) with the base population's @-metadata, and returns its path for the \
`genotype_file` parameter of the compute endpoints.",
    params(("name" = String, Path, description = "Upload name")),
    responses(
        (status = 200, description = "path, file and marker/individual counts", body = Envelope<serde_json::Value>),
        Unauthorized,
        (status = 404, description = "No such upload for this user", body = ApiError)
    )
)]
#[gnauth(token_only)]
pub async fn genotype_geno(
    ctx: Ctx,
    token: AuthToken,
    Path(name): Path<String>,
) -> Result<Json<Envelope<serde_json::Value>>, ApiError> {
    let rid = ctx.request_info();
    let config = Config::load();
    let owner = token.sub.clone();
    let dir = config.basket_dir.clone();
    let root = config.genotype_files_dir.clone();
    let wanted = name.clone();
    let base_version = {
        let owner = owner.clone();
        let dir = dir.clone();
        let wanted = name.clone();
        tokio::task::spawn_blocking(move || {
            genoupload::get(std::path::Path::new(&dir), &owner, &wanted)
        })
        .await
        .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone(), "genotype_geno"))?
        .map_err(|error| map_error(error, rid.clone(), "genotype_geno"))?
        .base_version
    };
    let stored = tokio::task::spawn_blocking({
        let dir = dir.clone();
        let owner = owner.clone();
        let wanted = wanted.clone();
        move || {
            // The base's labels travel with the file so R/qtl reads it as the
            // same kind of cross as the population it updates.
            let labels = genotypes::resolve_named(std::path::Path::new(&root), &base_version)
                .ok()
                .and_then(|path| genotypes::file_meta(&path).ok())
                .map(|meta| meta.labels)
                .unwrap_or_default();
            let path = genoupload::materialise(
                std::path::Path::new(&dir),
                &owner,
                &wanted,
                &labels,
            )?;
            let info = genoupload::get(std::path::Path::new(&dir), &owner, &wanted)?;
            Ok::<_, BasketError>((path, info))
        }
    })
    .await
    .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone(), "genotype_geno"))?
    .map_err(|error| map_error(error, rid.clone(), "genotype_geno"))?;
    let (path, info) = stored;
    let mut envelope = ctx
        .envelope(serde_json::json!({
            "name": info.name,
            "file": path.to_string_lossy(),
            "individuals": info.individuals,
            "markers": info.markers,
            "value_kind": info.value_kind,
            "digest": info.digest,
        }))
        .with_note("The file is written on demand, in the layout the R scripts here read; \
                    `file` is the path to pass as `genotype_file`.")
        .with_note("Nothing about the uploaded population is written to GeneNetwork's database.")
        .with_action(crate::response::SuggestedCall {
            method: "POST".to_string(),
            href: "/compute/qtl2".to_string(),
            description: "Map a trait over this genotype: pass `genotype_file` with the path \
                          above and the trait's values, or a `basket`."
                .to_string(),
            body: None,
        });
    envelope = envelope.with_links(crate::suggest::root_links());
    Ok(Json(envelope))
}

#[utoipa::path(delete, path = "/uploads/genotypes/{name}",
    summary = "Delete an uploaded genotype",
    params(("name" = String, Path, description = "Upload name")),
    responses(
        (status = 204, description = "Deleted"),
        Unauthorized,
        (status = 404, description = "No such upload for this user", body = ApiError)
    )
)]
#[gnauth(token_only)]
pub async fn delete_genotype(
    ctx: Ctx,
    token: AuthToken,
    Path(name): Path<String>,
) -> Result<StatusCode, ApiError> {
    let rid = ctx.request_info();
    let owner = token.sub.clone();
    let dir = Config::load().basket_dir;
    let wanted = name.clone();
    tokio::task::spawn_blocking(move || {
        genoupload::delete(std::path::Path::new(&dir), &owner, &wanted)
    })
    .await
    .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone(), "delete_genotype"))?
    .map_err(|error| map_error(error, rid.clone(), "delete_genotype"))?;
    Ok(StatusCode::NO_CONTENT)
}

/// Resolve `genotype_upload` for a compute endpoint: materialise the upload in
/// the form that engine reads, and hand back the path(s).
///
/// R/qtl and R/qtl2 read a GN `.geno`; GEMMA reads a BIMBAM pair. Either way
/// the caller names the upload and the server writes the file, so no user path
/// is ever passed around.
pub(crate) async fn resolve_upload(
    owner: String,
    basket_dir: String,
    genotype_dir: String,
    name: String,
    endpoint: &'static str,
    rid: RequestInfo,
    bimbam: bool,
) -> Result<(String, Option<String>), ApiError> {
    tokio::task::spawn_blocking(move || {
        let dir = std::path::Path::new(&basket_dir);
        let info = genoupload::get(dir, &owner, &name)?;
        // The base's @-metadata travels with the file so R/qtl reads the
        // upload as the same kind of cross as the population it updates.
        let labels = genotypes::resolve_named(
            std::path::Path::new(&genotype_dir),
            &info.base_version,
        )
        .ok()
        .and_then(|path| genotypes::file_meta(&path).ok())
        .map(|meta| meta.labels)
        .unwrap_or_default();
        if bimbam {
            let (geno, snps) = genoupload::materialise_bimbam(dir, &owner, &name, &labels)?;
            Ok::<_, BasketError>((
                geno.to_string_lossy().into_owned(),
                Some(snps.to_string_lossy().into_owned()),
            ))
        } else {
            let path = genoupload::materialise(dir, &owner, &name, &labels)?;
            Ok::<_, BasketError>((path.to_string_lossy().into_owned(), None))
        }
    })
    .await
    .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone(), endpoint))?
    .map_err(|error| map_error(error, rid, endpoint))
}