Derive URL column during ingest #66

Author: j23414

ingest/defaults/config.yaml

@@ -97,6 +97,14 @@ curate:
   output_id_field: "accession"
   # The field in the NDJSON record that contains the actual genomic sequence
   output_sequence_field: "sequence"
+  # The field in the NDJSON record that contains the actual GenBank accession
+  genbank_accession: 'accession'
+  # Added metadata columns
+  added_columns: [
+    'longitude',
+    'latitude'
+  ]
+
   # The list of metadata columns to keep in the final output of the curation pipeline.
   metadata_columns: [
     'accession',

ingest/rules/curate.smk

@@ -48,7 +48,7 @@ rule curate:
         annotations=config["curate"]["annotations"],
         manual_mapping="defaults/host_hostgenus_hosttype_map.tsv",
     output:
-        metadata="data/raw_metadata_curated.tsv",
+        metadata= "data/all_metadata.tsv",
         sequences="results/sequences.fasta",
     log:
         "logs/curate.txt",
@@ -68,7 +68,7 @@ rule curate:
         authors_default_value=config["curate"]["authors_default_value"],
         abbr_authors_field=config["curate"]["abbr_authors_field"],
         annotations_id=config["curate"]["annotations_id"],
-        metadata_columns=config["curate"]["metadata_columns"],
+        added_columns=config["curate"]["added_columns"],
         id_field=config["curate"]["output_id_field"],
         sequence_field=config["curate"]["output_sequence_field"],
     shell:
@@ -98,7 +98,7 @@ rule curate:
             | ./scripts/transform-state-names \
             | ./scripts/post_process_metadata.py \
             | ./scripts/add-field-names \
-                --metadata-columns {params.metadata_columns} \
+                --metadata-columns {params.added_columns} \
             | ./scripts/transform-new-fields \
                 --map-tsv {input.manual_mapping} \
                 --map-id host \
@@ -113,12 +113,31 @@ rule curate:
                 --output-id-field {params.id_field} \
                 --output-seq-field {params.sequence_field} ) 2>> {log}
         """
+rule add_metadata_columns:
+    """Add columns to metadata
+    Notable columns:
+    - [NEW] url: URL linking to the NCBI GenBank record ('https://www.ncbi.nlm.nih.gov/nuccore/*').
+    """
+    input:
+        metadata = "data/all_metadata.tsv"
+    output:
+        metadata = temp("data/all_metadata_added.tsv")
+    params:
+        accession=config['curate']['genbank_accession']
+    shell:
+        """
+        csvtk mutate2 -t \
+          -n url \
+          -e '"https://www.ncbi.nlm.nih.gov/nuccore/" + ${params.accession}' \
+          {input.metadata} \
+        > {output.metadata}
+        """
 
 rule subset_metadata:
     input:
-        metadata="data/raw_metadata_curated.tsv",
+        metadata="data/all_metadata_added.tsv",
     output:
-        metadata="data/raw_metadata.tsv",
+        metadata="data/subset_metadata.tsv",
     params:
         metadata_fields=",".join(config["curate"]["metadata_columns"]),
     shell:

ingest/rules/nextclade.smk

@@ -2,7 +2,7 @@
 This part of the workflow handles running Nextclade on the curated metadata
 and sequences.
 REQUIRED INPUTS:
-    metadata    = data/raw_metadata_all.tsv
+    metadata    = data/subset_metadata.tsv
     sequences   = data/sequences_all.fasta
     nextclade_datasets = ../nextclade/dataset
 OUTPUTS:
@@ -101,7 +101,7 @@ rule append_nextclade_columns:
     Append the nextclade results to the metadata
     """
     input:
-        metadata="data/raw_metadata.tsv",
+        metadata="data/subset_metadata.tsv",
         nextclade_subtypes="data/nextclade_clades.tsv",
     output:
         metadata_all="data/metadata_nextclade.tsv",

ingest/scripts/post_process_metadata.py

@@ -26,18 +26,12 @@ def _set_strain_name(record):
     )
 
 
-def _set_url(record):
-    """Set url column from accession"""
-    return "https://www.ncbi.nlm.nih.gov/nuccore/" + str(record["accession"])
-
-
 def main():
     args = parse_args()
 
     for index, record in enumerate(stdin):
         record = json.loads(record)
         record["strain"] = _set_strain_name(record)
-        record["url"] = _set_url(record)
         stdout.write(json.dumps(record) + "\n")
 
 

nextclade/rules/prepare_sequences.smk

@@ -3,7 +3,7 @@ This part of the workflow prepares sequences for constructing the phylogenetic t
 
 REQUIRED INPUTS:
 
-    metadata    = data/metadata.tsv
+    metadata    = data/all_metadata_added.tsv
     sequences   = data/sequences.fasta
     reference   = ../shared/reference.fasta
 
@@ -25,7 +25,7 @@ rule create_colors:
     message:
         "Creating custom color scale in {output.colors}"
     input:
-        metadata = "data/metadata_all.tsv"
+        metadata = "data/all_metadata_added.tsv"
     output:
         colors = "results/colors.tsv"
     log: