Skip to content

Commit a730317

Browse files
author
Michael Stromberg
committed
Merge branch 'develop'
2 parents 985c538 + a8b2bac commit a730317

File tree

242 files changed

+6737
-2228
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

242 files changed

+6737
-2228
lines changed

‎AnnotationLambda/AnnotationLambda.cs

+8-5
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@ public AnnotationResult Run(AnnotationConfig config, ILambdaContext context)
4646

4747
snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey);
4848

49-
string vcfUrl = config.vcfUrl;
50-
49+
string vcfUrl = config.vcfUrl;
50+
int variantCount = 0;
5151
using (var annotationResources = GetAnnotationResources(config))
5252
{
5353
if (annotationResources.InputStartVirtualPosition == -1) return GetSuccessOutput(result);
@@ -60,7 +60,7 @@ public AnnotationResult Run(AnnotationConfig config, ILambdaContext context)
6060
}
6161

6262
Logger.WriteLine("Scan for positions to preload complete.");
63-
63+
6464
using (var aes = new AesCryptoServiceProvider())
6565
{
6666
FileMetadata jsonMetadata, jasixMetadata;
@@ -84,7 +84,7 @@ public AnnotationResult Run(AnnotationConfig config, ILambdaContext context)
8484

8585
using (var jsonCompressStream = new BlockGZipStream(jsonMd5Stream, CompressionMode.Compress))
8686
{
87-
StreamAnnotation.Annotate(headerStream, inputVcfStream, jsonCompressStream, jasixMd5Stream, annotationResources, vcfFilter, true);
87+
variantCount = StreamAnnotation.Annotate(headerStream, inputVcfStream, jsonCompressStream, jasixMd5Stream, annotationResources, vcfFilter, true).variantCount;
8888
}
8989

9090
Logger.WriteLine("Annotation done.");
@@ -107,6 +107,7 @@ public AnnotationResult Run(AnnotationConfig config, ILambdaContext context)
107107
LambdaUtilities.DeleteTempOutput();
108108
if (string.IsNullOrEmpty(result.filePath)) throw new FileNotFoundException();
109109

110+
result.variantCount = variantCount;
110111
return GetSuccessOutput(result);
111112
}
112113
catch (Exception exception)
@@ -155,7 +156,9 @@ private static AnnotationResources GetAnnotationResources(AnnotationConfig annot
155156
var genomeAssembly = GenomeAssemblyHelper.Convert(annotationConfig.genomeAssembly);
156157
string cachePathPrefix = LambdaUrlHelper.GetCacheFolder().UrlCombine(genomeAssembly.ToString()).UrlCombine(LambdaUrlHelper.DefaultCacheSource);
157158
string nirvanaS3Ref = LambdaUrlHelper.GetRefUrl(genomeAssembly);
158-
string saManifestUrl = LambdaUtilities.GetManifestUrl(annotationConfig.supplementaryAnnotations ?? "latest", genomeAssembly);
159+
// SaVersion will be provided as an environment variable. Defaults to "latest"
160+
string saVersion = Environment.GetEnvironmentVariable("SaVersion");
161+
string saManifestUrl = LambdaUtilities.GetManifestUrl(string.IsNullOrEmpty(saVersion)? "latest": saVersion, genomeAssembly);
159162
var metrics = new PerformanceMetrics();
160163

161164
var annotationResources = new AnnotationResources(nirvanaS3Ref, cachePathPrefix,

‎AnnotationLambda/AnnotationLambda.csproj

+7-7
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22
<PropertyGroup>
3-
<TargetFramework>netcoreapp2.1</TargetFramework>
3+
<TargetFramework>netcoreapp3.1</TargetFramework>
44
<GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
55
<AWSProjectType>Lambda</AWSProjectType>
6-
<OutputPath>bin\$(Configuration)</OutputPath>
6+
<OutputPath>bin\$(Configuration)</OutputPath>
77
<LangVersion>latest</LangVersion>
88
</PropertyGroup>
99
<ItemGroup>
10-
<PackageReference Include="Amazon.Lambda.Core" Version="1.1.0" />
11-
<PackageReference Include="Amazon.Lambda.Serialization.Json" Version="1.7.0" />
12-
<PackageReference Include="AWSSDK.Lambda" Version="3.3.106.4" />
13-
<PackageReference Include="AWSSDK.S3" Version="3.3.109.4" />
14-
<PackageReference Include="AWSSDK.SimpleNotificationService" Version="3.3.101.100" />
10+
<PackageReference Include="Amazon.Lambda.Core" Version="2.0.0" />
11+
<PackageReference Include="Amazon.Lambda.Serialization.Json" Version="2.0.0" />
12+
<PackageReference Include="AWSSDK.Lambda" Version="3.7.0.15" />
13+
<PackageReference Include="AWSSDK.S3" Version="3.7.0.17" />
14+
<PackageReference Include="AWSSDK.SimpleNotificationService" Version="3.7.1.1" />
1515
</ItemGroup>
1616
<ItemGroup>
1717
<ProjectReference Include="..\Cloud\Cloud.csproj" />

‎AnnotationLambda/Properties/launchSettings.json

-18
This file was deleted.

‎CacheUtils/CacheUtils.csproj

+5-5
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22
<PropertyGroup>
33
<OutputType>Exe</OutputType>
4-
<TargetFramework>netcoreapp2.1</TargetFramework>
5-
<OutputPath>..\bin\$(Configuration)</OutputPath>
4+
<TargetFramework>netcoreapp3.1</TargetFramework>
5+
<OutputPath>..\bin\$(Configuration)</OutputPath>
66
<LangVersion>latest</LangVersion>
77
</PropertyGroup>
88
<Import Project="..\CommonAssemblyInfo.props" />
@@ -12,9 +12,9 @@
1212
</Content>
1313
</ItemGroup>
1414
<ItemGroup>
15-
<PackageReference Include="Microsoft.Extensions.Configuration" Version="3.1.0" />
16-
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="3.1.0" />
17-
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="3.1.0" />
15+
<PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
16+
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
17+
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="5.0.0" />
1818
</ItemGroup>
1919
<ItemGroup>
2020
<ProjectReference Include="..\CommandLine\CommandLine.csproj" />

‎CacheUtils/Properties/launchSettings.json

-49
This file was deleted.

‎CacheUtils/TranscriptCache/TranscriptCacheUtilities.cs

+4-4
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,13 @@ public static List<ITranscript> GetTranscripts(DataBundle bundle, IChromosomeInt
2020

2121
public static IntervalArray<T>[] ToIntervalArrays<T>(this IEnumerable<T> items, int numRefSeqs) where T : IChromosomeInterval
2222
{
23-
var intervalArrays = new IntervalArray<T>[numRefSeqs];
24-
var itemsByRef = items.GetMultiValueDict(x => x.Chromosome.Index);
23+
var intervalArrays = new IntervalArray<T>[numRefSeqs];
24+
Dictionary<ushort, List<T>> itemsByRef = items.GetMultiValueDict(x => x.Chromosome.Index);
2525

2626
foreach (ushort refIndex in itemsByRef.Keys.OrderBy(x => x))
2727
{
28-
var unsortedItems = itemsByRef[refIndex];
29-
var intervals = unsortedItems.OrderBy(x => x.Start).ThenBy(x => x.End).ToIntervals(unsortedItems.Count);
28+
List<T> unsortedItems = itemsByRef[refIndex];
29+
Interval<T>[] intervals = unsortedItems.OrderBy(x => x.Start).ThenBy(x => x.End).ToIntervals(unsortedItems.Count);
3030
intervalArrays[refIndex] = new IntervalArray<T>(intervals);
3131
}
3232

‎Cloud/Cloud.csproj

+6-6
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22
<PropertyGroup>
3-
<TargetFramework>netcoreapp2.1</TargetFramework>
3+
<TargetFramework>netcoreapp3.1</TargetFramework>
44
<OutputPath>..\bin\$(Configuration)</OutputPath>
55
<LangVersion>latest</LangVersion>
66
</PropertyGroup>
77
<ItemGroup>
8-
<PackageReference Include="Amazon.Lambda.Core" Version="1.1.0" />
9-
<PackageReference Include="Amazon.Lambda.Serialization.Json" Version="1.7.0" />
10-
<PackageReference Include="AWSSDK.S3" Version="3.3.109.4" />
11-
<PackageReference Include="AWSSDK.SimpleNotificationService" Version="3.3.101.100" />
12-
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
8+
<PackageReference Include="Amazon.Lambda.Core" Version="2.0.0" />
9+
<PackageReference Include="Amazon.Lambda.Serialization.Json" Version="2.0.0" />
10+
<PackageReference Include="AWSSDK.S3" Version="3.7.0.17" />
11+
<PackageReference Include="AWSSDK.SimpleNotificationService" Version="3.7.1.1" />
12+
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
1313
</ItemGroup>
1414
<ItemGroup>
1515
<ProjectReference Include="..\Genome\Genome.csproj" />

‎Cloud/Messages/Annotation/AnnotationConfig.cs

+8-9
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,14 @@ public sealed class AnnotationConfig
66
{
77
// ReSharper disable InconsistentNaming
88
// ReSharper disable NotAccessedField.Global
9-
public string id;
10-
public string genomeAssembly;
11-
public string vcfUrl;
12-
public string tabixUrl;
13-
public S3Path outputDir;
14-
public string outputPrefix;
15-
public string supplementaryAnnotations;
16-
public List<SaUrls> customAnnotations;
17-
public string customStrUrl;
9+
public string id;
10+
public string genomeAssembly;
11+
public string vcfUrl;
12+
public string tabixUrl;
13+
public S3Path outputDir;
14+
public string outputPrefix;
15+
public List<SaUrls> customAnnotations;
16+
public string customStrUrl;
1817
public AnnotationRange annotationRange;
1918
// ReSharper restore NotAccessedField.Global
2019
// ReSharper restore InconsistentNaming

‎Cloud/Messages/Annotation/AnnotationResult.cs

+2
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ public sealed class AnnotationResult
1111
public string status;
1212
public string filePath;
1313
public ErrorCategory? errorCategory;
14+
15+
public int variantCount;
1416
// ReSharper restore UnassignedField.Global
1517
// ReSharper restore InconsistentNaming
1618
}

‎Cloud/Messages/Custom/CustomConfig.cs

+6-3
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,12 @@ namespace Cloud.Messages.Custom
44
{
55
public sealed class CustomConfig
66
{
7-
public string id;
8-
public string tsvUrl;
9-
public S3Path outputDir;
7+
public string id;
8+
public string tsvUrl;
9+
public S3Path outputDir;
10+
public JwtFields jwtFields;
11+
1012
public bool skipGeneIdValidation;
13+
public bool skipRefBaseValidation;
1114
}
1215
}

‎Cloud/Messages/Custom/CustomResult.cs

+7-5
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,13 @@
44
// ReSharper disable InconsistentNaming
55
public sealed class CustomResult
66
{
7-
public string id;
8-
public string status;
9-
public string genomeAssembly;
10-
public FileList created;
11-
public bool noValidEntries;
7+
public string id;
8+
public string status;
9+
public string genomeAssembly;
10+
public FileList created;
11+
public bool noValidEntries;
12+
public JwtFields jwtFields;
13+
public int variantCount;
1214
}
1315
// ReSharper restore InconsistentNaming
1416
// ReSharper restore NotAccessedField.Global

‎Cloud/Messages/JwtFields.cs

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
namespace Cloud.Messages
2+
{
3+
public class JwtFields
4+
{
5+
// ReSharper disable InconsistentNaming
6+
public string authorizedParty;
7+
public string subject;
8+
public string tenantId;
9+
10+
// ReSharper restore InconsistentNaming
11+
}
12+
}

‎Cloud/Messages/Nirvana/NirvanaConfig.cs

+3-2
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,11 @@ public sealed class NirvanaConfig
1212
public string vcfUrl;
1313
public string tabixUrl;
1414
public S3Path outputDir;
15-
public string supplementaryAnnotations;
1615
// ReSharper disable once UnassignedField.Global
1716
public List<SaUrls> customAnnotations;
18-
public string customStrUrl;
17+
public string customStrUrl;
18+
public JwtFields jwtFields;
19+
1920
// ReSharper restore InconsistentNaming
2021

2122
internal void CheckRequiredFieldsNotNull()

‎Cloud/Messages/Nirvana/NirvanaResult.cs

+6-3
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,12 @@ public sealed class NirvanaResult
44
{
55
// ReSharper disable InconsistentNaming
66
// ReSharper disable NotAccessedField.Global
7-
public string id;
8-
public string status;
9-
public FileList created;
7+
public string id;
8+
public string status;
9+
public FileList created;
10+
public JwtFields jwtFields;
11+
public int variantCount;
12+
1013
// ReSharper restore NotAccessedField.Global
1114
// ReSharper restore InconsistentNaming
1215
}

‎CommandLine/CommandLine.csproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22
<PropertyGroup>
3-
<TargetFramework>netcoreapp2.1</TargetFramework>
3+
<TargetFramework>netcoreapp3.1</TargetFramework>
44
<OutputPath>..\bin\$(Configuration)</OutputPath>
55
<LangVersion>latest</LangVersion>
66
</PropertyGroup>

‎CommonAssemblyInfo.props

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
22
<PropertyGroup>
33
<Company>Illumina</Company>
4-
<Copyright>© 2020 Illumina, Inc.</Copyright>
5-
<FileVersion>3.13.0</FileVersion>
6-
<AssemblyVersion>3.13.0</AssemblyVersion>
7-
<Version>3.13.0</Version>
4+
<Copyright>© 2021 Illumina, Inc.</Copyright>
5+
<FileVersion>3.15.0</FileVersion>
6+
<AssemblyVersion>3.15.0</AssemblyVersion>
7+
<Version>3.15.0</Version>
88
<Authors>Stromberg, Roy, Lajugie, Jiang, Li, and Kang</Authors>
99
</PropertyGroup>
1010
</Project>

‎Compression/Compression.csproj

100644100755
+2-5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22
<PropertyGroup>
3-
<TargetFramework>netcoreapp2.1</TargetFramework>
3+
<TargetFramework>netcoreapp3.1</TargetFramework>
44
<OutputPath>..\bin\$(Configuration)</OutputPath>
55
<LangVersion>latest</LangVersion>
66
</PropertyGroup>
@@ -12,9 +12,6 @@
1212
<None Include="Packages\BlockCompression\BlockCompression.dll" Link="BlockCompression.dll">
1313
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
1414
</None>
15-
<None Include="Packages\BlockCompression\BlockCompression.pdb" Link="BlockCompression.pdb">
16-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
17-
</None>
1815
<None Include="Packages\BlockCompression\libBlockCompression.so" Link="libBlockCompression.so">
1916
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
2017
</None>
@@ -23,4 +20,4 @@
2320
</None>
2421
</ItemGroup>
2522
<Import Project="..\CommonAssemblyInfo.props" />
26-
</Project>
23+
</Project>

‎Compression/Packages/BlockCompression/BlockCompression.dll

28 KB
Binary file not shown.

‎Compression/Packages/BlockCompression/BlockCompression.pdb

-1.93 MB
Binary file not shown.

‎Compression/Packages/BlockCompression/libBlockCompression.dylib

142 KB
Binary file not shown.

‎Compression/Packages/BlockCompression/libBlockCompression.so

-80.7 KB
Binary file not shown.

0 commit comments

Comments
 (0)