diff --git a/docs/.vscode/settings.json b/docs/.vscode/settings.json index ba0beebe5..ec6e05ee8 100644 --- a/docs/.vscode/settings.json +++ b/docs/.vscode/settings.json @@ -2,10 +2,17 @@ "cSpell.words": [ "Analysing", "Ecoacoustics", + "Ecosounds", "executables", "parallelise", "recognisers", "recognizers", "visualise" - ] + ], + "markdownlint.config": { + "MD028": false, + "MD025": { + "front_matter_title": "" + } + } } \ No newline at end of file diff --git a/docs/images/download-batch.png b/docs/images/download-batch.png new file mode 100644 index 000000000..057d09b30 --- /dev/null +++ b/docs/images/download-batch.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aaf89c06210ca83c5cc4e24c91e9f9c448b585d9e1ddef16fa489c04801cddaf +size 101650 diff --git a/docs/images/download-files.png b/docs/images/download-files.png new file mode 100644 index 000000000..dc6b8bdce --- /dev/null +++ b/docs/images/download-files.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:881db3821e8e6b41bc450b48176ba27813781a4f124e0b3fb3eac84429347e0c +size 81176 diff --git a/docs/images/download-repositories.png b/docs/images/download-repositories.png new file mode 100644 index 000000000..506cb0109 --- /dev/null +++ b/docs/images/download-repositories.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3c27f277c8d0bf291d54da4ef0cf0881a62f2b1fc36eeb44e5f332e0a13b9c3 +size 8815 diff --git a/docs/images/download-search.png b/docs/images/download-search.png new file mode 100644 index 000000000..202e6000e --- /dev/null +++ b/docs/images/download-search.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:290715012cc735e690ad7bda8f50eee3880b299924d4c4a8b2f6d8c95f9c69c9 +size 397868 diff --git a/docs/images/download.png b/docs/images/download.png new file mode 100644 index 000000000..bd9346c8f --- /dev/null +++ b/docs/images/download.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:615bead706c721ad8c98b1fa88d9f0bec4dc2c8141071ea2618d3c354b3f266a +size 18179 diff --git a/docs/technical/commands/download.md b/docs/technical/commands/download.md new file mode 100644 index 000000000..f425c03ff --- /dev/null +++ b/docs/technical/commands/download.md @@ -0,0 +1,34 @@ +--- +title: Download +uid: command-download +--- + +# Download + +- **Command**: `download` +- **Config file**: none (no config file required) + +This command shows what sub-commands are available for the `download` command. + +## Usage + +```shell +$ AnalysisPrograms.exe download +``` + +![download files screenshot](~/images/download.png) + +## Options + +```shell +Downloads audio from a repository + + +Commands: + batch Download a multiple files from a remote repository + file Download a single file from a remote repository + repositories Lists available repositories which we can download from + search Preview which files would be downloaded by the batch command +``` + +This command has no options. See the sub-commands for more information. \ No newline at end of file diff --git a/docs/technical/commands/download/batch.md b/docs/technical/commands/download/batch.md new file mode 100644 index 000000000..03dd38260 --- /dev/null +++ b/docs/technical/commands/download/batch.md @@ -0,0 +1,75 @@ +--- +title: Download Batch +uid: command-download-batch +--- + +# Download File + +- **Command**: `download file` +- **Config file**: none (no config file required) + +This command downloads one or more files from a remote Acoustic Workbench server. +You can use the command to preview which files would be downloaded by the batch command. + + +## Usage + +```shell +$ AnalysisPrograms.exe download batch [options] +``` + +Here is an example of a command line with abbreviated path names: + +```shell +$ AnalysisPrograms.exe download batch -s 1 -s 2 --start '2021-01-24' --end '2021-12-25' --repository="A2O" --auth-token "REDACTED" --output "D:\Temp\downloads" +``` + +![download batch screenshot](~/images/download-batch.png) + +This command downloads all the recordings from sites `1` and `2` which were recorded between `2021-01-24` and `2021-12-25`. + +You'll need to log in using an authentication token. You can get one by logging in to the website and clicking on the "My Account" link. + +## Options + +```shell +Download a single file from a remote repository + +Usage: AnalysisPrograms.exe download batch [options] + +Arguments: + Ids One or more audio files to download + +Options: + -p|--project-ids Project IDs to filter recordings by + -r|--region-ids Region IDs to filter recordings by + -s|--site-ids Site IDs to filter recordings by + --start A date (inclusive) to filter out recordings. Can parse an ISO8601 date. + --end A date (exclusive) to filter out recordings. Can parse an ISO8601 date. + -f|--flat If used will not place downloaded files into sub-folders + -o|--output A directory to write output to + -repo|--repository Which repository to use to download audio from + -a|--auth-token Your personal access token for the repository + ...... +``` + +- `-p|--project-ids `: Project IDs to filter recordings by. +- `-r|--region-ids `: Region IDs to filter recordings by. +- `-s|--site-ids `: Site IDs to filter recordings by. +- `--start `: A date (inclusive) to filter out recordings. Can parse an ISO8601 date. +- `--end `: A date (exclusive) to filter out recordings. Can parse an ISO8601 date. +- `-f|--flat`: If used will not place downloaded files into sub-folders. Normally recordings are split into sub-folders by their site name. +- `-o|--output `: A directory to put the downloaded audio recordings into. +- `-repo|--repository `: Which repository to use to download audio from. Either `A2O` or `Ecosounds` +- `-a|--auth-token `: Your personal access token for the repository. + +The options `--repo` and `--auth-token` are required. + +You can only choose one of `--project-ids`, `--region-ids`, or `--site-ids` per command. +But for each you specify the option multiple times. For example, to search multiple sites, you can do this: + +``` +... -p 123 -p 456 -p 789 ... +``` + +If you specify a start date (`--start`) then you must also include an end date. diff --git a/docs/technical/commands/download/file.md b/docs/technical/commands/download/file.md new file mode 100644 index 000000000..1a94773a6 --- /dev/null +++ b/docs/technical/commands/download/file.md @@ -0,0 +1,58 @@ +--- +title: Download File +uid: command-download-file +--- + +# Download File + +- **Command**: `download file` +- **Config file**: none (no config file required) + +This command downloads one or more files from a remote Acoustic Workbench server. + +The files to download are identified by their unique identifiers. +You can find the ID for any recording by looking at the URL of the recording, +or by looking on the details page of a recording. + +## Usage + +```shell +$ AnalysisPrograms.exe download [file_ids...] [options] +``` + +Here is an example of a command line: + +```shell +$ AnalysisPrograms.exe download file 1 2 3 4 5 7 --repository="A2O" --auth-token "REDACTED" --output "D:\Temp\download" +``` + +![download files screenshot](~/images/download-files.png) + +This command will download the files with the IDs `1`, `2`, `3`, `4`, `5`, and `7`. + +You'll need to log in using an authentication token. You can get one by logging in to the website and clicking on the "My Account" link. + +## Options + +```shell +Download a single file from a remote repository + +Usage: AnalysisPrograms.exe download file [options] + +Arguments: + Ids One or more audio files to download + +Options: + -f|--flat If used will not place downloaded files into sub-folders + -o|--output A directory to write output to + -repo|--repository Which repository to use to download audio from + -a|--auth-token Your personal access token for the repository + ...... +``` + +- `-f|--flat`: If used will not place downloaded files into sub-folders. Normally recordings are split into sub-folders by their site name. +- `-o|--output `: A directory to put the downloaded audio recordings into. +- `-repo|--repository `: Which repository to use to download audio from. Either `A2O` or `Ecosounds` +- `-a|--auth-token `: Your personal access token for the repository. + +All options except for `--flat` are required. \ No newline at end of file diff --git a/docs/technical/commands/download/repositories.md b/docs/technical/commands/download/repositories.md new file mode 100644 index 000000000..b298eedbc --- /dev/null +++ b/docs/technical/commands/download/repositories.md @@ -0,0 +1,23 @@ +--- +title: Download Repositories +uid: command-download-repositories +--- + +# Download Repositories + +- **Command**: `download repositories` +- **Config file**: none (no config file required) + +This list the available repositories which we can download from. + +## Usage + +```shell +$ AnalysisPrograms.exe download repositories +``` + +![download repositories screenshot](~/images/download-repositories.png) + +## Options + +This command has not options. \ No newline at end of file diff --git a/docs/technical/commands/download/search.md b/docs/technical/commands/download/search.md new file mode 100644 index 000000000..489eb7be2 --- /dev/null +++ b/docs/technical/commands/download/search.md @@ -0,0 +1,68 @@ +--- +title: Download Search +uid: command-download-search +--- + +# Download File + +- **Command**: `download search` +- **Config file**: none (no config file required) + +This command searches a repository for recordings. + +You can use this command to see which recordings would be downloaded by the command. + +## Usage + +```shell +$ AnalysisPrograms.exe download search [options] +``` + +Here is an example of a command line: + +```shell +$ AnalysisPrograms.exe download search -s 1 -s 2 --start '2021-01-24' --end '2021-12-25' --repository="A2O" --auth-token "REDACTED" +``` + +![download search screenshot](~/images/download-search.png) + +This command shows all the recordings from sites `1` and `2` which were recorded between `2021-01-24` and `2021-12-25`. + +You'll need to log in using an authentication token. You can get one by logging in to the website and clicking on the "My Account" link. + +## Options + +```shell +Preview which files would be downloaded by the batch command + +Usage: AnalysisPrograms.exe download search [options] + +Options: + -p|--project-ids Project IDs to filter recordings by + -r|--region-ids Region IDs to filter recordings by + -s|--site-ids Site IDs to filter recordings by + --start A date (inclusive) to filter out recordings. Can parse an ISO8601 date. + --end A date (exclusive) to filter out recordings. Can parse an ISO8601 date. + -repo|--repository Which repository to use to download audio from + -a|--auth-token Your personal access token for the repository + ...... +``` + +- `-p|--project-ids `: Project IDs to filter recordings by. +- `-r|--region-ids `: Region IDs to filter recordings by. +- `-s|--site-ids `: Site IDs to filter recordings by. +- `--start `: A date (inclusive) to filter out recordings. Can parse an ISO8601 date. +- `--end `: A date (exclusive) to filter out recordings. Can parse an ISO8601 date. +- `-repo|--repository `: Which repository to use to download audio from. Either `A2O` or `Ecosounds` +- `-a|--auth-token `: Your personal access token for the repository. + +The options `--repo` and `--auth-token` are required. + +You can only choose one of `--project-ids`, `--region-ids`, or `--site-ids` per command. +But for each you specify the option multiple times. For example, to search multiple sites, you can do this: + +``` +... -p 123 -p 456 -p 789 ... +``` + +If you specify a start date (`--start`) then you must also include an end date. diff --git a/docs/technical/commands/toc.yml b/docs/technical/commands/toc.yml index a14669cc1..70723cbda 100644 --- a/docs/technical/commands/toc.yml +++ b/docs/technical/commands/toc.yml @@ -6,3 +6,14 @@ href: draw_false_colour_spectrogram.md - name: Concatenate Index Files href: concatenate_index_files.md +- name: Download + href: download.md + items: + - name: Repositories + href: download/repositories.md + - name: File + href: download/file.md + - name: Search + href: download/search.md + - name: Batch + href: download/batch.md \ No newline at end of file diff --git a/src/AcousticWorkbench/AcousticWorkbench.csproj b/src/AcousticWorkbench/AcousticWorkbench.csproj index aed25b22c..10e136d82 100644 --- a/src/AcousticWorkbench/AcousticWorkbench.csproj +++ b/src/AcousticWorkbench/AcousticWorkbench.csproj @@ -10,4 +10,8 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + + + + \ No newline at end of file diff --git a/src/AcousticWorkbench/AcousticWorkbenchListResponse.cs b/src/AcousticWorkbench/AcousticWorkbenchListResponse.cs new file mode 100644 index 000000000..54bc929ce --- /dev/null +++ b/src/AcousticWorkbench/AcousticWorkbenchListResponse.cs @@ -0,0 +1,12 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AcousticWorkbench +{ + public class AcousticWorkbenchListResponse + : AcousticWorkbenchResponse + { + public T[] Data { get; set; } + } +} \ No newline at end of file diff --git a/src/AcousticWorkbench/AcousticWorkbenchResponse{T}.cs b/src/AcousticWorkbench/AcousticWorkbenchResponse{T}.cs index f42328b6d..5f319af82 100644 --- a/src/AcousticWorkbench/AcousticWorkbenchResponse{T}.cs +++ b/src/AcousticWorkbench/AcousticWorkbenchResponse{T}.cs @@ -4,49 +4,9 @@ namespace AcousticWorkbench { - using System; - using System.Collections.Generic; - using Newtonsoft.Json; - using Newtonsoft.Json.Linq; - // [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] - // public abstract class RubyJsonObject - // { - // } - - public class AcousticWorkbenchResponse + public abstract class AcousticWorkbenchResponse { public Meta Meta { get; set; } - - public T Data { get; set; } - } - - public class Meta - { - public string Status { get; set; } - - public string Message { get; set; } - - public Error Error { get; set; } - - public override string ToString() - { - return $"[Status: {this.Status}] {this.Message}\n" + - (this.Error?.ToString() ?? string.Empty); - } - } - - public class Error - { - public string Details { get; set; } - - public Dictionary Links { get; set; } - - public JRaw Info { get; set; } - - public override string ToString() - { - return "API error: " + this.Details + Environment.NewLine + (this.Info.ToString(Formatting.Indented) ?? string.Empty); - } } } \ No newline at end of file diff --git a/src/AcousticWorkbench/AcousticWorkbenchSingleResponse.cs b/src/AcousticWorkbench/AcousticWorkbenchSingleResponse.cs new file mode 100644 index 000000000..aa3c2796f --- /dev/null +++ b/src/AcousticWorkbench/AcousticWorkbenchSingleResponse.cs @@ -0,0 +1,12 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AcousticWorkbench +{ + public class AcousticWorkbenchSingleResponse + : AcousticWorkbenchResponse + { + public T Data { get; set; } + } +} \ No newline at end of file diff --git a/src/AcousticWorkbench/Api.cs b/src/AcousticWorkbench/Api.cs index 4ebb19c6a..c203be7d7 100644 --- a/src/AcousticWorkbench/Api.cs +++ b/src/AcousticWorkbench/Api.cs @@ -16,7 +16,7 @@ static Api() { Default = new Api() { - Host = "www.ecosounds.org", + Host = "api.ecosounds.org", Protocol = "https", Version = DefaultVersion, }; diff --git a/src/AcousticWorkbench/AudioRecordingService.cs b/src/AcousticWorkbench/AudioRecordingService.cs deleted file mode 100644 index 0190a6df7..000000000 --- a/src/AcousticWorkbench/AudioRecordingService.cs +++ /dev/null @@ -1,26 +0,0 @@ -// -// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). -// - -namespace AcousticWorkbench -{ - using System.Threading.Tasks; - using AcousticWorkbench.Models; - - public class AudioRecordingService : Service - { - public AudioRecordingService(IAuthenticatedApi authenticatedApi) - : base(authenticatedApi) - { - } - - public async Task GetAudioRecording(long audioRecordingId) - { - var uri = this.AuthenticatedApi.GetAudioRecordingUri(audioRecordingId); - - var response = await this.HttpClient.GetAsync(uri); - - return await this.ProcessApiResult(response); - } - } -} \ No newline at end of file diff --git a/src/AcousticWorkbench/Filter.cs b/src/AcousticWorkbench/Filter.cs new file mode 100644 index 000000000..da67e2b51 --- /dev/null +++ b/src/AcousticWorkbench/Filter.cs @@ -0,0 +1,130 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AcousticWorkbench +{ + using System; + using System.Collections.Immutable; + using System.Linq; + using Acoustics.Shared; + using Newtonsoft.Json; + using Newtonsoft.Json.Converters; + + public partial record QueryFilter(Projection Projection, ImmutableDictionary Filter, Sorting Sorting); + + public partial record QueryFilter + { + public static QueryFilter Empty { get; } = new(new(), ImmutableDictionary.Empty, null); + } + + public record Projection(ImmutableArray? Include = default, ImmutableArray? Exclude = default); + + public record Sorting(string OrderBy, Direction Direction); + + [Serializable] + [JsonConverter(typeof(StringEnumConverter))] + public enum Direction + { + Desc, + Asc, + } + + public static class FilterExtensions + { + public static QueryFilter WithProjectionInclude(this QueryFilter filter, params string[] fields) + { + return filter with + { + Projection = filter.Projection with + { + Include = ApiNaming(fields) + } + }; + } + + public static QueryFilter WithProjectionExclude(this QueryFilter filter, params string[] fields) + { + return filter with + { + Projection = filter.Projection with + { + Exclude = ApiNaming(fields) + } + }; + } + + public static QueryFilter FilterById(this QueryFilter filter, long id, string name = "id") + { + var result = filter.Filter.Add( + ApiNaming(name), + Pairs(("eq", id))); + + return filter with + { + Filter = result + }; + } + + public static QueryFilter FilterByIds(this QueryFilter filter, string name = "id", params ulong[] ids) + { + if (ids is null or { Length: 0 }) + { + return filter; + } + + var result = filter.Filter.Add( + ApiNaming(name), + Pairs(("in", ids))); + + return filter with + { + Filter = result + }; + } + + public static QueryFilter FilterByRange(this QueryFilter filter, string field, Interval? range) + where T : struct, IComparable, IFormattable + { + if (range is null or { IsEmpty: true }) + { + return filter; + } + + var result = filter.Filter.Add( + ApiNaming(field), + Pairs( + ("range", + Pairs( + (ApiNaming("interval"), range.Value.ToString(true)))))); + + return filter with + { + Filter = result + }; + } + + public static QueryFilter OrderBy(this QueryFilter filter, string field, Direction direction = Direction.Asc) + { + return filter with + { + Sorting = new Sorting(ApiNaming(field), direction) + }; + } + + private static ImmutableDictionary Pairs(params (string Key, object Value)[] pairs) + { + return pairs.ToImmutableDictionary(pairs => pairs.Key, pairs => pairs.Value); + } + + private static ImmutableArray ApiNaming(string[] names) + { + return names.Select(x => Service.NamingStrategy.GetPropertyName(x, hasSpecifiedName: false)).ToImmutableArray(); + } + + private static string ApiNaming(string name) + { + return Service.NamingStrategy.GetPropertyName(name, hasSpecifiedName: false); + } + } +} diff --git a/src/AcousticWorkbench/IApi.cs b/src/AcousticWorkbench/IApi.cs index 13d184e96..a6f69dd35 100644 --- a/src/AcousticWorkbench/IApi.cs +++ b/src/AcousticWorkbench/IApi.cs @@ -11,7 +11,5 @@ public interface IApi string Version { get; } string Protocol { get; } - - //string Uri { get; } } } \ No newline at end of file diff --git a/src/AcousticWorkbench/IWebsite.cs b/src/AcousticWorkbench/IWebsite.cs new file mode 100644 index 000000000..7f2e62613 --- /dev/null +++ b/src/AcousticWorkbench/IWebsite.cs @@ -0,0 +1,13 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AcousticWorkbench +{ + public interface IWebsite + { + string Host { get; } + + string Protocol { get; } + } +} \ No newline at end of file diff --git a/src/AcousticWorkbench/Models/AudioEvent.cs b/src/AcousticWorkbench/Models/AudioEvent.cs new file mode 100644 index 000000000..ff13f6cc7 --- /dev/null +++ b/src/AcousticWorkbench/Models/AudioEvent.cs @@ -0,0 +1,36 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AcousticWorkbench.Models +{ + using System; + using System.Collections.Generic; + + public class AudioEvent : IModelWithMeta + { + public long Id { get; set; } + + public long AudioRecordingId { get; set; } + + public double StartTimeSeconds { get; set; } + + public double EndTimeSeconds { get; set; } + + public double LowFrequencyHertz { get; set; } + + public double HighFrequencyHertz { get; set; } + + public bool IsReference { get; set; } + + public long CreatorId { get; set; } + + public DateTimeOffset CreatedAt { get; set; } + + public DateTimeOffset UpdatedAt { get; set; } + + public IList Taggings { get; set; } + + public Meta Meta { get; set; } + } +} \ No newline at end of file diff --git a/src/AcousticWorkbench/Models/AudioRecording.cs b/src/AcousticWorkbench/Models/AudioRecording.cs new file mode 100644 index 000000000..94ac6163b --- /dev/null +++ b/src/AcousticWorkbench/Models/AudioRecording.cs @@ -0,0 +1,45 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AcousticWorkbench.Models +{ + using System; + using Newtonsoft.Json; + + public class AudioRecording : IModelWithMeta + { + public long Id { get; set; } + + public string Uuid { get; set; } + + public DateTimeOffset RecordedDate { get; set; } + + public long SiteId { get; set; } + + public double DurationSeconds { get; set; } + + public int SampleRateHertz { get; set; } + + public int Channels { get; set; } + + public int BitRateBps { get; set; } + + public string MediaType { get; set; } + + public long DataLengthBytes { get; set; } + + public string Status { get; set; } + + public DateTimeOffset CreatedAt { get; set; } + + public DateTimeOffset UpdatedAt { get; set; } + + public string CanonicalFileName { get; set; } + + [JsonProperty("sites.name")] + public string SiteName { get; set; } + + public Meta Meta { get; set; } + } +} \ No newline at end of file diff --git a/src/AcousticWorkbench/Models/Capability.cs b/src/AcousticWorkbench/Models/Capability.cs new file mode 100644 index 000000000..2220994aa --- /dev/null +++ b/src/AcousticWorkbench/Models/Capability.cs @@ -0,0 +1,12 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AcousticWorkbench +{ + public class Capability + { + public bool? Can { get; set; } + public string Details { get; set; } + } +} \ No newline at end of file diff --git a/src/AcousticWorkbench/Models/Error.cs b/src/AcousticWorkbench/Models/Error.cs new file mode 100644 index 000000000..5d83bfa0f --- /dev/null +++ b/src/AcousticWorkbench/Models/Error.cs @@ -0,0 +1,25 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AcousticWorkbench +{ + using System; + using System.Collections.Generic; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + + public class Error + { + public string Details { get; set; } + + public Dictionary Links { get; set; } + + public JRaw Info { get; set; } + + public override string ToString() + { + return "API error: " + this.Details + Environment.NewLine + (this.Info?.ToString(Formatting.Indented) ?? string.Empty); + } + } +} \ No newline at end of file diff --git a/src/AcousticWorkbench/Models/IModelWithMeta.cs b/src/AcousticWorkbench/Models/IModelWithMeta.cs new file mode 100644 index 000000000..36007625a --- /dev/null +++ b/src/AcousticWorkbench/Models/IModelWithMeta.cs @@ -0,0 +1,24 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AcousticWorkbench.Models +{ + public interface IModelWithMeta + { + Meta Meta { get; set; } + + bool? Can(string action) + { + if (this.Meta?.Capabilities is { Count: > 0 }) + { + if (this.Meta.Capabilities.ContainsKey(action)) + { + return this.Meta.Capabilities[action].Can; + } + } + + return null; + } + } +} \ No newline at end of file diff --git a/src/AcousticWorkbench/Models/Media.cs b/src/AcousticWorkbench/Models/Media.cs new file mode 100644 index 000000000..8e9d623f3 --- /dev/null +++ b/src/AcousticWorkbench/Models/Media.cs @@ -0,0 +1,59 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AcousticWorkbench.Models +{ + using System.Collections.Generic; + + public class Media + { + public Recording Recording { get; set; } + + public CommonParametersModel CommonParameters { get; set; } + + public AvailableModel Available { get; set; } + + public class CommonParametersModel + { + public double StartOffset { get; set; } + + public double EndOffset { get; set; } + + public long AudioEventId { get; set; } + + public int Channel { get; set; } + + public int SampleRate { get; set; } + } + + public class FormatInfo + { + public string MediaType { get; set; } + + public string Extension { get; set; } + + public string Url { get; set; } + } + + public class ImageFormatInfo : FormatInfo + { + public int WindowSize { get; set; } + + public string WindowFunction { get; set; } + + public string Colour { get; set; } + + public double Ppms { get; set; } + } + + public class AvailableModel + { + public Dictionary Audio { get; set; } + + public Dictionary Image { get; set; } + + public Dictionary Text { get; set; } + } + } +} \ No newline at end of file diff --git a/src/AcousticWorkbench/Models/Meta.cs b/src/AcousticWorkbench/Models/Meta.cs new file mode 100644 index 000000000..dba963a42 --- /dev/null +++ b/src/AcousticWorkbench/Models/Meta.cs @@ -0,0 +1,27 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AcousticWorkbench +{ + using System.Collections.Generic; + + public class Meta + { + public string Status { get; set; } + + public string Message { get; set; } + + public Error Error { get; set; } + + public Paging Paging { get; set; } + + public Dictionary Capabilities { get; set; } + + public override string ToString() + { + return $"[Status: {this.Status}] {this.Message}\n" + + (this.Error?.ToString() ?? string.Empty); + } + } +} \ No newline at end of file diff --git a/src/AcousticWorkbench/Models/Models.cs b/src/AcousticWorkbench/Models/Models.cs deleted file mode 100644 index 444fcd30d..000000000 --- a/src/AcousticWorkbench/Models/Models.cs +++ /dev/null @@ -1,148 +0,0 @@ -// -// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). -// - -namespace AcousticWorkbench.Models -{ - using System; - using System.Collections.Generic; - - public class AudioRecording - { - public long Id { get; set; } - - public string Uuid { get; set; } - - public DateTimeOffset RecordedDate { get; set; } - - public long SiteId { get; set; } - - public double DurationSeconds { get; set; } - - public int SampleRateHertz { get; set; } - - public int Channels { get; set; } - - public int BitRateBps { get; set; } - - public string MediaType { get; set; } - - public long DataLengthBytes { get; set; } - - public string Status { get; set; } - - public DateTimeOffset CreatedAt { get; set; } - - public DateTimeOffset UpdatedAt { get; set; } - } - - public class Tagging - { - public long Id { get; set; } - - public long AudioEventId { get; set; } - - public long TagId { get; set; } - - public long CreatorId { get; set; } - - public long UpdaterId { get; set; } - - public DateTimeOffset CreatedAt { get; set; } - - public DateTimeOffset UpdatedAt { get; set; } - } - - public class AudioEvent - { - public long Id { get; set; } - - public long AudioRecordingId { get; set; } - - public double StartTimeSeconds { get; set; } - - public double EndTimeSeconds { get; set; } - - public double LowFrequencyHertz { get; set; } - - public double HighFrequencyHertz { get; set; } - - public bool IsReference { get; set; } - - public long CreatorId { get; set; } - - public DateTimeOffset CreatedAt { get; set; } - - public DateTimeOffset UpdatedAt { get; set; } - - public IList Taggings { get; set; } - } - - public class Recording - { - public long Id { get; set; } - - public string Uuid { get; set; } - - public DateTimeOffset RecordedDate { get; set; } - - public double DurationSeconds { get; set; } - - public int SampleRateHertz { get; set; } - - public int ChannelCount { get; set; } - - public string MediaType { get; set; } - } - - public class CommonParameters - { - public double StartOffset { get; set; } - - public double EndOffset { get; set; } - - public long AudioEventId { get; set; } - - public int Channel { get; set; } - - public int SampleRate { get; set; } - } - - public class FormatInfo - { - public string MediaType { get; set; } - - public string Extension { get; set; } - - public string Url { get; set; } - } - - public class ImageFormatInfo : FormatInfo - { - public int WindowSize { get; set; } - - public string WindowFunction { get; set; } - - public string Colour { get; set; } - - public double Ppms { get; set; } - } - - public class Available - { - public Dictionary Audio { get; set; } - - public Dictionary Image { get; set; } - - public Dictionary Text { get; set; } - } - - public class Media - { - public Recording Recording { get; set; } - - public CommonParameters CommonParameters { get; set; } - - public Available Available { get; set; } - } -} \ No newline at end of file diff --git a/src/AcousticWorkbench/Models/Paging.cs b/src/AcousticWorkbench/Models/Paging.cs new file mode 100644 index 000000000..3a653441b --- /dev/null +++ b/src/AcousticWorkbench/Models/Paging.cs @@ -0,0 +1,17 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AcousticWorkbench +{ + public class Paging + { + public int Page { get; set; } + + public int Items { get; set; } + + public int Total { get; set; } + + public int MaxPage { get; set; } + } +} \ No newline at end of file diff --git a/src/AcousticWorkbench/Models/Recording.cs b/src/AcousticWorkbench/Models/Recording.cs new file mode 100644 index 000000000..9bbd90741 --- /dev/null +++ b/src/AcousticWorkbench/Models/Recording.cs @@ -0,0 +1,25 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AcousticWorkbench.Models +{ + using System; + + public class Recording + { + public long Id { get; set; } + + public string Uuid { get; set; } + + public DateTimeOffset RecordedDate { get; set; } + + public double DurationSeconds { get; set; } + + public int SampleRateHertz { get; set; } + + public int ChannelCount { get; set; } + + public string MediaType { get; set; } + } +} \ No newline at end of file diff --git a/src/AcousticWorkbench/Models/Tagging.cs b/src/AcousticWorkbench/Models/Tagging.cs new file mode 100644 index 000000000..2dcbc37ca --- /dev/null +++ b/src/AcousticWorkbench/Models/Tagging.cs @@ -0,0 +1,25 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AcousticWorkbench.Models +{ + using System; + + public class Tagging + { + public long Id { get; set; } + + public long AudioEventId { get; set; } + + public long TagId { get; set; } + + public long CreatorId { get; set; } + + public long UpdaterId { get; set; } + + public DateTimeOffset CreatedAt { get; set; } + + public DateTimeOffset UpdatedAt { get; set; } + } +} \ No newline at end of file diff --git a/src/AcousticWorkbench/Service.cs b/src/AcousticWorkbench/Service.cs index fc430f275..82b7b90f6 100644 --- a/src/AcousticWorkbench/Service.cs +++ b/src/AcousticWorkbench/Service.cs @@ -5,11 +5,13 @@ namespace AcousticWorkbench { using System; + using System.Collections.Generic; using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Text; using System.Threading.Tasks; + using AcousticWorkbench.Models; using Newtonsoft.Json; using Newtonsoft.Json.Serialization; @@ -26,11 +28,13 @@ public abstract class Service /// public static readonly TimeSpan ClientTimeout = TimeSpan.FromSeconds(120 + (Environment.ProcessorCount * 10)); + public static readonly NamingStrategy NamingStrategy = new SnakeCaseNamingStrategy(); + private const string ApplicationJson = "application/json"; private readonly DefaultContractResolver defaultContractResolver = new DefaultContractResolver() { - NamingStrategy = new SnakeCaseNamingStrategy(), + NamingStrategy = NamingStrategy, }; private readonly JsonSerializerSettings jsonSerializerSettings; @@ -47,13 +51,18 @@ static Service() protected Service(IApi api) { - this.HttpClient = new HttpClient(); - - this.HttpClient.Timeout = ClientTimeout; + this.HttpClient = new HttpClient + { + Timeout = ClientTimeout, + }; this.HttpClient.DefaultRequestHeaders.Accept.Add(MediaTypeWithQualityHeaderValue.Parse(ApplicationJson)); this.HttpClient.BaseAddress = api.Base(); - this.jsonSerializerSettings = new JsonSerializerSettings { ContractResolver = this.defaultContractResolver }; + this.jsonSerializerSettings = new JsonSerializerSettings + { + ContractResolver = this.defaultContractResolver, + NullValueHandling = NullValueHandling.Ignore, + }; } protected Service(IAuthenticatedApi authenticatedApi) @@ -88,19 +97,24 @@ protected StringContent SerializeContent(object obj, out string serializedString return new StringContent(serializedString, Encoding.UTF8, ApplicationJson); } - protected AcousticWorkbenchResponse Deserialize(string json) + protected AcousticWorkbenchSingleResponse DeserializeSingle(string json) + { + return JsonConvert.DeserializeObject>(json, this.jsonSerializerSettings); + } + + protected AcousticWorkbenchListResponse DeserializeList(string json) { - return JsonConvert.DeserializeObject>(json, this.jsonSerializerSettings); + return JsonConvert.DeserializeObject>(json, this.jsonSerializerSettings); } protected async Task ProcessApiResult(HttpResponseMessage response, string requestBody = "") { var json = await response.Content.ReadAsStringAsync(); - AcousticWorkbenchResponse result = null; + AcousticWorkbenchSingleResponse result = null; try { - result = this.Deserialize(json); + result = this.DeserializeSingle(json); } catch (JsonReaderException) { @@ -122,15 +136,58 @@ protected async Task ProcessApiResult(HttpResponseMessage response, string throw new InvalidOperationException("Service has a null data blob that was not caught by error handling"); } + // tag the models with Meta if possible + if (result.Data is IModelWithMeta model) + { + model.Meta = result.Meta; + } + return result.Data; } - public class HttpResponseException : Exception + protected async Task> ProcessApiResults(HttpResponseMessage response, string requestBody = "") { - public HttpResponseMessage Response { get; } + var json = await response.Content.ReadAsStringAsync(); - public Meta ResponseMeta { get; } + AcousticWorkbenchListResponse result = null; + try + { + result = this.DeserializeList(json); + } + catch (JsonReaderException) + { + // throw if it was meant to work... if not then we're already in an error case... best effort to get to + // error handling block below. + if (response.IsSuccessStatusCode) + { + throw; + } + } + if (!response.IsSuccessStatusCode) + { + throw new HttpResponseException(response, result?.Meta, requestBody); + } + + if (result == null) + { + throw new InvalidOperationException("Service has a null data blob that was not caught by error handling"); + } + + // tag the models with Meta if possible + foreach (var item in result.Data) + { + if (item is IModelWithMeta model) + { + model.Meta = result.Meta; + } + } + + return result.Data; + } + + public class HttpResponseException : Exception + { public HttpResponseException(HttpResponseMessage response, Meta responseMeta, string requestBody = "") { this.Response = response; @@ -142,6 +199,10 @@ public HttpResponseException(HttpResponseMessage response, Meta responseMeta, st $"API meta: {responseMeta}"; } + public HttpResponseMessage Response { get; } + + public Meta ResponseMeta { get; } + public override string Message { get; } } } diff --git a/src/AcousticWorkbench/AcousticEventService.cs b/src/AcousticWorkbench/Services/AcousticEventService.cs similarity index 74% rename from src/AcousticWorkbench/AcousticEventService.cs rename to src/AcousticWorkbench/Services/AcousticEventService.cs index 2787945ec..bc714ff56 100644 --- a/src/AcousticWorkbench/AcousticEventService.cs +++ b/src/AcousticWorkbench/Services/AcousticEventService.cs @@ -22,21 +22,12 @@ public async Task GetAudioEvent(long audioEventId) var uri = this.AuthenticatedApi.GetAudioEventFilterUri(); var body = this.SerializeContent( - new - { - filter = new - { - id = new - { - eq = audioEventId, - }, - }, - }, + QueryFilter.Empty.FilterById(audioEventId), out var stringBody); var response = await this.HttpClient.PostAsync(uri, body); - var audioEvents = await this.ProcessApiResult(response, stringBody); + var audioEvents = await this.ProcessApiResults(response, stringBody); return audioEvents.Single(); } diff --git a/src/AcousticWorkbench/Services/AudioRecordingService.cs b/src/AcousticWorkbench/Services/AudioRecordingService.cs new file mode 100644 index 000000000..e67e34fd2 --- /dev/null +++ b/src/AcousticWorkbench/Services/AudioRecordingService.cs @@ -0,0 +1,130 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AcousticWorkbench +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.IO; + using System.Net.Http; + using System.Threading.Tasks; + using Acoustics.Shared; + using AcousticWorkbench.Models; + + public partial class AudioRecordingService : Service + { + public AudioRecordingService(IAuthenticatedApi authenticatedApi) + : base(authenticatedApi) + { + } + + public async Task GetAudioRecording(long audioRecordingId) + { + var uri = this.AuthenticatedApi.GetAudioRecordingUri(audioRecordingId); + + var response = await this.HttpClient.GetAsync(uri); + + return await this.ProcessApiResult(response); + } + + public async Task> FilterRecordingsForDownload( + ulong[] ids = null, + Interval? range = null, + ulong[] projectIds = null, + ulong[] regionIds = null, + ulong[] siteIds = null, + int page = 1) + { + var uri = this.AuthenticatedApi.GetAudioRecordingFilterUri(page); + + var filter = QueryFilter.Empty + .FilterByIds("id", ids) + .FilterByIds("projects.id", projectIds) + .FilterByIds("regions.id", regionIds) + .FilterByIds("sites.id", siteIds) + .FilterByRange("recorded_date", range) + .WithProjectionInclude("id", "recorded_date", "sites.name", "site_id", "canonical_file_name") + .OrderBy("recorded_date"); + + var body = this.SerializeContent( + filter, + out var stringBody); + + var response = await this.HttpClient.PostAsync(uri, body); + + return await this.ProcessApiResults(response, stringBody); + } + + public async Task DownloadOriginalAudioRecording( + long id, + string destination, + Action totalCallback = null, + Action progressCallback = null) + { + var totalTime = Stopwatch.StartNew(); + var headers = Stopwatch.StartNew(); + var url = this.AuthenticatedApi.GetOriginalAudioUri(id); + + var message = new HttpRequestMessage() + { + Method = HttpMethod.Get, + RequestUri = url, + }; + + // unset the default accept type + message.Headers.Accept.Clear(); + + // get just the headers + using HttpResponseMessage response = await this.HttpClient.GetAsync( + url, + HttpCompletionOption.ResponseHeadersRead); + + headers.Stop(); + + if (!response.IsSuccessStatusCode) + { + // only called for errors - this call should always throw + await this.ProcessApiResult(response); + } + + // Set the max value of bytes + totalCallback(response.Content.Headers.ContentLength ?? 0); + + var body = Stopwatch.StartNew(); + + ulong total = 0; + using (var contentStream = await response.Content.ReadAsStreamAsync()) + using (var fileStream = new FileStream(destination, FileMode.Create, FileAccess.Write, FileShare.None, 8192, useAsync: true)) + { + var buffer = new byte[8192]; + while (true) + { + var read = await contentStream.ReadAsync(buffer); + if (read == 0) + { + break; + } + + // Increment the number of read bytes for the progress task + progressCallback(read); + + // Write the read bytes to the output stream + await fileStream.WriteAsync(buffer.AsMemory(0, read)); + total += (ulong)read; + } + } + + body.Stop(); + totalTime.Stop(); + + return new DownloadStats( + destination, + totalTime.Elapsed, + headers.Elapsed, + body.Elapsed, + total); + } + } +} \ No newline at end of file diff --git a/src/AcousticWorkbench/AuthenticationService.cs b/src/AcousticWorkbench/Services/AuthenticationService.cs similarity index 97% rename from src/AcousticWorkbench/AuthenticationService.cs rename to src/AcousticWorkbench/Services/AuthenticationService.cs index 7a34d3199..fd4791977 100644 --- a/src/AcousticWorkbench/AuthenticationService.cs +++ b/src/AcousticWorkbench/Services/AuthenticationService.cs @@ -56,7 +56,7 @@ public async Task Login(string username, string password) private async Task ProcessResult(HttpResponseMessage response) { var json = await response.Content.ReadAsStringAsync(); - var result = this.Deserialize(json); + var result = this.DeserializeSingle(json); if (!response.IsSuccessStatusCode) { diff --git a/src/AcousticWorkbench/Services/DownloadStats.cs b/src/AcousticWorkbench/Services/DownloadStats.cs new file mode 100644 index 000000000..ae61b89ce --- /dev/null +++ b/src/AcousticWorkbench/Services/DownloadStats.cs @@ -0,0 +1,13 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AcousticWorkbench +{ + using System; + + public partial class AudioRecordingService + { + public record DownloadStats(string File, TimeSpan Total, TimeSpan Headers, TimeSpan Body, ulong Bytes); + } +} \ No newline at end of file diff --git a/src/AcousticWorkbench/MediaService.cs b/src/AcousticWorkbench/Services/MediaService.cs similarity index 100% rename from src/AcousticWorkbench/MediaService.cs rename to src/AcousticWorkbench/Services/MediaService.cs diff --git a/src/AcousticWorkbench/UrlGenerator.cs b/src/AcousticWorkbench/UrlGenerator.cs index e769c255f..9adbe066e 100644 --- a/src/AcousticWorkbench/UrlGenerator.cs +++ b/src/AcousticWorkbench/UrlGenerator.cs @@ -8,14 +8,26 @@ namespace AcousticWorkbench public static class UrlGenerator { + public static string GetPage(int? page = null) + { + return page switch + { + null => string.Empty, + < 1 => throw new ArgumentException("msut be greater or equal to 1", nameof(page)), + int i => "page=" + i, + }; + } + public static Uri GetAudioEventUri(this IApi api, long audioRecordingId, long audioEventId) { return api.Base($"audio_recordings/{audioRecordingId}/audio_events/{audioEventId}"); } - public static Uri GetAudioEventFilterUri(this IApi api) + public static Uri GetAudioEventFilterUri(this IApi api, int? page = null) { - return api.Base("audio_events/filter"); + var uri = api.Base("audio_events/filter", GetPage(page)); + + return uri; } public static Uri GetAudioRecordingUri(this IApi api, long audioRecordingId) @@ -23,6 +35,21 @@ public static Uri GetAudioRecordingUri(this IApi api, long audioRecordingId) return api.Base($"audio_recordings/{audioRecordingId}"); } + public static Uri GetAudioRecordingViewUri(this IWebsite api, long audioRecordingId) + { + return api.ViewBase($"audio_recordings/{audioRecordingId}"); + } + + public static Uri GetAudioRecordingFilterUri(this IApi api, int? page = null) + { + return api.Base($"audio_recordings/filter", GetPage(page)); + } + + public static Uri GetOriginalAudioUri(this IApi api, long audioRecordingId) + { + return api.Base($"audio_recordings/{audioRecordingId}/original"); + } + public static Uri GetLoginUri(this IApi api) { return api.Base("security"); @@ -33,7 +60,7 @@ public static Uri GetSessionValidateUri(this IApi api) return api.Base("security/user"); } - public static Uri GetListenUri(this IApi api, long audioRecordingId, double startOffsetSeconds, double? endOffsetSeconds = null) + public static Uri GetListenUri(this IWebsite api, long audioRecordingId, double startOffsetSeconds, double? endOffsetSeconds = null) { string end = endOffsetSeconds == null ? string.Empty : $"&end ={endOffsetSeconds}"; return api.ViewBase($"listen/{audioRecordingId}?start={startOffsetSeconds}{end}"); @@ -57,14 +84,14 @@ public static Uri GetMediaWaveUri( int? sampleRate = null, byte? channel = 0) { - return api.Base( - $"audio_recordings/{audioRecordingId}/media.wav?" - + $"start_offset={startOffsetSeconds}&end_offset={endOffsetSeconds}" - + (sampleRate.HasValue ? $"&sample_rate={sampleRate.Value}" : string.Empty) - + (channel.HasValue ? $"&channel={channel.Value}" : string.Empty)); + var query = $"?start_offset={startOffsetSeconds}&end_offset={endOffsetSeconds}" + + (sampleRate.HasValue ? $"&sample_rate={sampleRate.Value}" : string.Empty) + + (channel.HasValue ? $"&channel={channel.Value}" : string.Empty); + + return api.Base($"audio_recordings/{audioRecordingId}/media.wav", query); } - public static Uri Base(this IApi api, string path = "") + public static Uri Base(this IApi api, string path = "", string query = "") { string version = api.Version == "v1" ? string.Empty : "/" + api.Version; @@ -73,10 +100,19 @@ public static Uri Base(this IApi api, string path = "") path = "/" + path; } - return new Uri($"{api.Protocol}://{api.Host}{version}{path}"); + query = query switch + { + null => string.Empty, + "" => query, + _ when query.StartsWith("&") => "?" + query[1..], + _ when !query.StartsWith("?") => "?" + query, + _ => query, + }; + + return new Uri($"{api.Protocol}://{api.Host}{version}{path}{query}"); } - public static Uri ViewBase(this IApi api, string path = "") + public static Uri ViewBase(this IWebsite api, string path = "") { if (path.Length > 0) { diff --git a/src/AcousticWorkbench/Website.cs b/src/AcousticWorkbench/Website.cs new file mode 100644 index 000000000..d6ef4f311 --- /dev/null +++ b/src/AcousticWorkbench/Website.cs @@ -0,0 +1,45 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AcousticWorkbench +{ + using System; + + public record Website : IWebsite + { + private string protocol; + + public string Host { get; init; } + + public Website(string host, string protocol) + { + this.Host = host; + this.Protocol = protocol; + } + + public static Website Parse(string uri) + { + var (success, message) = Api.TryParse(uri, out var api); + if (success) + { + return new Website(api.Host, api.Protocol); + } + + throw new ArgumentException("Cannot parse Acoustic Workbench API string: " + message); + } + + public string Protocol + { + get => this.protocol; init + { + if (value != "https") + { + throw new ArgumentException($"{nameof(this.Protocol)} only supports https"); + } + + this.protocol = value; + } + } + } +} \ No newline at end of file diff --git a/src/Acoustics.Shared/PathUtils.cs b/src/Acoustics.Shared/PathUtils.cs index 073578e4a..36d9b53dd 100644 --- a/src/Acoustics.Shared/PathUtils.cs +++ b/src/Acoustics.Shared/PathUtils.cs @@ -10,9 +10,14 @@ namespace Acoustics.Shared using System.IO; using System.Runtime.InteropServices; using System.Text; + using System.Text.RegularExpressions; public static class PathUtils { + public static readonly Regex SafeFilenameRegex = new Regex( + "[^-_A-Za-z0-9.]+", + RegexOptions.Compiled); + private const int MaxPath = byte.MaxValue; private static readonly HashSet UnsafeChars = new HashSet(Path.GetInvalidPathChars()); @@ -39,6 +44,11 @@ public static bool HasUnicodeOrUnsafeChars(string path) return false; } + public static string MakeSafeFilename(string input) + { + return SafeFilenameRegex.Replace(input, string.Empty); + } + /// /// Gets the short 8.3 filename for a file. /// diff --git a/src/AnalysisBase/IHasStatus.cs b/src/AnalysisBase/IHasStatus.cs index 64b336bce..2996acd9e 100644 --- a/src/AnalysisBase/IHasStatus.cs +++ b/src/AnalysisBase/IHasStatus.cs @@ -1,5 +1,5 @@ // -------------------------------------------------------------------------------------------------------------------- -// +// // All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). // // diff --git a/src/AnalysisPrograms/AcousticWorkbench.Orchestration/Repositories.cs b/src/AnalysisPrograms/AcousticWorkbench.Orchestration/Repositories.cs new file mode 100644 index 000000000..69a0d2e0b --- /dev/null +++ b/src/AnalysisPrograms/AcousticWorkbench.Orchestration/Repositories.cs @@ -0,0 +1,33 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AnalysisPrograms.AcousticWorkbench.Orchestration +{ + using System; + using System.Collections.Generic; + using System.Linq; + using global::AcousticWorkbench; + + public class Repositories + { + public static readonly IReadOnlyCollection Known = new[] + { + new Repository( + "A2O", + Api.Parse("https://api.acousticobservatory.org/"), + Website.Parse("https://data.acousticobservatory.org/"), + "https://www.acousticobservatory.org/"), + new Repository( + "Ecosounds", + Api.Parse("https://api.ecosounds.org/"), + Website.Parse("https://ecosounds.org/"), + "https://ecosounds.org/"), + }; + + public static Repository Find(string repository) + { + return Known.FirstOrDefault(x => x.Name.Equals(repository, StringComparison.InvariantCultureIgnoreCase)); + } + } +} diff --git a/src/AnalysisPrograms/AcousticWorkbench.Orchestration/Repository.cs b/src/AnalysisPrograms/AcousticWorkbench.Orchestration/Repository.cs new file mode 100644 index 000000000..75ea44ea3 --- /dev/null +++ b/src/AnalysisPrograms/AcousticWorkbench.Orchestration/Repository.cs @@ -0,0 +1,10 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AnalysisPrograms.AcousticWorkbench.Orchestration +{ + using global::AcousticWorkbench; + + public record Repository(string Name, Api Api, Website Website, string HomeUri); +} \ No newline at end of file diff --git a/src/AnalysisPrograms/AnalysisPrograms.csproj b/src/AnalysisPrograms/AnalysisPrograms.csproj index 64705c4bc..be22187a9 100644 --- a/src/AnalysisPrograms/AnalysisPrograms.csproj +++ b/src/AnalysisPrograms/AnalysisPrograms.csproj @@ -25,10 +25,11 @@ - + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/AnalysisPrograms/Download/BatchCommand.cs b/src/AnalysisPrograms/Download/BatchCommand.cs new file mode 100644 index 000000000..ca23c8e83 --- /dev/null +++ b/src/AnalysisPrograms/Download/BatchCommand.cs @@ -0,0 +1,43 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AnalysisPrograms.Download +{ + using System.IO; + using System.Threading.Tasks; + using AnalysisPrograms.Production; + using AnalysisPrograms.Production.Validation; + using McMaster.Extensions.CommandLineUtils; + + [Command(BatchCommandName, Description = "Download a multiple files from a remote repository")] + public class BatchCommand : SearchCommand + { + private const string BatchCommandName = "batch"; + + [Option( + Description = "If used will not place downloaded files into sub-folders")] + public override bool Flat { get; set; } + + [Option(Description = "A directory to write output to")] + [DirectoryExistsOrCreate(createIfNotExists: false)] + [LegalFilePath] + public override DirectoryInfo Output { get; set; } + + public override async Task Execute(CommandLineApplication app) + { + this.ValidateBatchOptions(); + + this.ShowOptions(); + + await this.SignIn(); + + var results = await this.DownloadFiles(); + + this.PrintSummary(results); + + return ExceptionLookup.Ok; + } + } + +} \ No newline at end of file diff --git a/src/AnalysisPrograms/Download/DownloadBaseCommand.cs b/src/AnalysisPrograms/Download/DownloadBaseCommand.cs new file mode 100644 index 000000000..af9c43c60 --- /dev/null +++ b/src/AnalysisPrograms/Download/DownloadBaseCommand.cs @@ -0,0 +1,228 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AnalysisPrograms.Download +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Threading.Tasks; + using Acoustics.Shared; + using global::AcousticWorkbench; + using global::AcousticWorkbench.Models; + using log4net; + using McMaster.Extensions.CommandLineUtils; + using Spectre.Console; + using static global::AcousticWorkbench.AudioRecordingService; + + + public abstract class DownloadBaseCommand : RemoteRepositoryBase + { + private static readonly ILog Log = LogManager.GetLogger(typeof(DownloadBaseCommand)); + + private AudioRecordingService audioRecordingService; + + public int MaxPage { get; private set; } + + public int Total { get; private set; } = 0; + + public int PageIndex { get; private set; } = 0; + + public virtual bool Flat { get; set; } = false; + + public virtual DirectoryInfo Output { get; set; } + + private AudioRecordingService AudioRecordingService + { + get + { + if (this.Api is null) + { + throw new InvalidOperationException(); + } + + if (this.audioRecordingService == null) + { + this.audioRecordingService = new AudioRecordingService(this.Api); + } + + return this.audioRecordingService; + } + } + + public void ValidateCommon() + { + this.Console.MarkupLine("Preparing to download..."); + + this.ValidateRepository(); + + if (this.Output is null) + { + var working = Directory.GetCurrentDirectory(); + this.Console.WarnLine($"{nameof(this.Output)} was not supplied, using the current working directory {working}"); + this.Output = working.ToDirectoryInfo(); + } + else if (!this.Output.Exists) + { + Log.Verbose($"Creating {this.Output}"); + this.Output.Create(); + } + + this.ValidateAuthToken(); + } + + protected override void AddOptionToShowOptions(Grid grid) + { + grid.AddRow(nameof(this.Output), this.Output.FullName) + .AddRow(nameof(this.Flat), this.Flat.ToString()); + } + + protected async Task> GetPage(ProgressContext context, int? nextPage = null) + { + if (nextPage is null) + { + this.PageIndex++; + } + else + { + this.PageIndex = nextPage.Value; + } + + ProgressTask task = context.AddTask($"Searching for page {this.PageIndex}...").IsIndeterminate(); + task.StartTask(); + var page = await this.PageQuery(this.AudioRecordingService, this.PageIndex); + task.StopTask(); + + if (page.Count == 0) + { + return page; + } + + if (this.Total is 0) + { + var first = page.First(); + var paging = first.Meta.Paging; + this.Total = paging.Total; + this.MaxPage = paging.MaxPage; + this.Console.MarkupLine($"[lime]{paging.Total}[/] recordings found"); + } + + return page; + } + + protected abstract Task> PageQuery(AudioRecordingService service, int page); + + protected async Task> DownloadFiles() + { + var progress = this.Console.Progress() + .HideCompleted(true) + .Columns( + new TaskDescriptionColumn(), + new ProgressBarColumn(), + new PercentageColumn(), + new RemainingTimeColumn(), + new TransferSpeedColumn()); + + var results = await progress.StartAsync(async progress => + { + var page = await this.GetPage(progress); + this.Console.WriteLine("\n"); + + var totalTask = progress.AddTask($"Downloading [green]{this.Total}[/] recordings", maxValue: this.Total); + + var results = new List(this.Total); + while (page.Count > 0) + { + foreach (var recording in page) + { + var downloadTask = progress.AddTask($"Downloading [yellow]{recording.Id}[/]"); + + var result = await this.DownloadFile(downloadTask, recording); + + totalTask.Increment(1); + var shortName = recording.CanonicalFileName.Truncate(this.DescriptionWidth(), "...", true); + this.Console.MarkupLine($"Downloaded [yellow]{shortName}[/] in [blue]{result.Total}[/]"); + results.Add(result); + } + + // just keep fetching pages until no results returned + page = await this.GetPage(progress); + } + + totalTask.StopTask(); + return results; + }); + + return results; + } + + protected void PrintSummary(IEnumerable results) + { + var totalTime = TimeSpan.Zero; + var waitingTime = TimeSpan.Zero; + var count = 0; + var totalBytes = 0ul; + foreach (var stat in results) + { + totalTime += stat.Total; + waitingTime += stat.Headers; + count++; + totalBytes += stat.Bytes; + } + + var speed = totalBytes / totalTime.TotalSeconds; + + Grid lastGrid = new Grid() + .AddColumn(new GridColumn().NoWrap().PadRight(4)) + .AddColumn() + .AddRow("Files downloaded", count.ToString()) + .AddRow("Time taken", $"[blue]{totalTime}[/]") + .AddRow("Time waiting", $"[blue]{waitingTime}[/]") + .AddRow("Total bytes", new FileSize(totalBytes).ToString()) + .AddRow("Average speed", new FileSize(speed).ToString() + "/s"); + + this.Console.Write(new Panel(lastGrid).Header("Summary")); + + this.Console.MarkupLine($"Files downloaded to folder [yellow]{this.Output}[/]"); + this.Console.SuccessLine("Completed"); + } + + private static string FolderName(AudioRecording recording) + { + return recording.SiteId + "_" + PathUtils.MakeSafeFilename(recording.SiteName); + } + + private int DescriptionWidth() + { + const int reservedWidth = 40; + return this.Console.Profile.Width - reservedWidth; + } + + private async Task DownloadFile(ProgressTask progress, AudioRecording recording) + { + var destinationDirectory = this.Output; + if (!this.Flat) + { + destinationDirectory = destinationDirectory.Combine(FolderName(recording)); + } + + destinationDirectory.Create(); + + var destination = destinationDirectory.CombinePath(recording.CanonicalFileName); + + // Start the progress task + progress.StartTask(); + + var result = await this.AudioRecordingService.DownloadOriginalAudioRecording( + recording.Id, + destination, + (total) => progress.MaxValue = total, + (increment) => progress.Increment(increment)); + + progress.StopTask(); + return result; + } + } +} \ No newline at end of file diff --git a/src/AnalysisPrograms/Download/DownloadCommand.cs b/src/AnalysisPrograms/Download/DownloadCommand.cs new file mode 100644 index 000000000..08d6273ee --- /dev/null +++ b/src/AnalysisPrograms/Download/DownloadCommand.cs @@ -0,0 +1,27 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AnalysisPrograms.Download +{ + using System.Threading.Tasks; + using AnalysisPrograms.Production.Arguments; + using McMaster.Extensions.CommandLineUtils; + + [Command(DownloadCommandName, Description = "Downloads audio from a repository")] + [Subcommand(typeof(RepositoriesCommand))] + [Subcommand(typeof(FileCommand))] + [Subcommand(typeof(SearchCommand))] + [Subcommand(typeof(BatchCommand))] + public class DownloadCommand : SubCommandBase + { + public const string DownloadCommandName = "download"; + + public override Task Execute(CommandLineApplication app) + { + MainEntry.PrintUsage("Choose a sub command.", MainEntry.Usages.Node, DownloadCommandName); + + return this.Ok(); + } + } +} \ No newline at end of file diff --git a/src/AnalysisPrograms/Download/FileCommand.cs b/src/AnalysisPrograms/Download/FileCommand.cs new file mode 100644 index 000000000..dec049b65 --- /dev/null +++ b/src/AnalysisPrograms/Download/FileCommand.cs @@ -0,0 +1,74 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AnalysisPrograms.Download +{ + using System; + using System.Collections.Generic; + using System.ComponentModel.DataAnnotations; + using System.IO; + using System.Linq; + using System.Threading.Tasks; + using AnalysisPrograms.Production; + using AnalysisPrograms.Production.Validation; + using global::AcousticWorkbench; + using global::AcousticWorkbench.Models; + using McMaster.Extensions.CommandLineUtils; + using Spectre.Console; + + [Command(FileCommandName, Description = "Download a single file from a remote repository", ExtendedHelpText = ExtendedHelp)] + public class FileCommand : DownloadBaseCommand + { + private const string FileCommandName = "file"; + + private const string ExtendedHelp = @$" +Each file argument must be of the form of an integer ID for the audio recording you want to download. + +{RepositoriesHint}"; + + [Option( + Description = "If used will not place downloaded files into sub-folders")] + public override bool Flat { get; set; } + + [Option(Description = "A directory to write output to")] + [DirectoryExistsOrCreate(createIfNotExists: false)] + [LegalFilePath] + public override DirectoryInfo Output { get; set; } + + [Argument( + 0, + Description = "One or more audio files to download")] + [Required] + public ulong[] Ids { get; set; } + + public override async Task Execute(CommandLineApplication app) + { + this.ValidateCommon(); + + this.ShowOptions(); + + await this.SignIn(); + + this.Console.WriteLine("Starting downloads..."); + + var results = await this.DownloadFiles(); + + this.PrintSummary(results); + + return ExceptionLookup.Ok; + } + + protected override void AddOptionToShowOptions(Grid grid) + { + grid.AddRow(nameof(this.Ids), this.Ids.Join(", ")); + base.AddOptionToShowOptions(grid); + } + + protected override Task> PageQuery(AudioRecordingService service, int page) + { + return service.FilterRecordingsForDownload(ids: this.Ids, page: page); + } + } + +} \ No newline at end of file diff --git a/src/AnalysisPrograms/Download/RemoteRepositoryBase.cs b/src/AnalysisPrograms/Download/RemoteRepositoryBase.cs new file mode 100644 index 000000000..d43d3708a --- /dev/null +++ b/src/AnalysisPrograms/Download/RemoteRepositoryBase.cs @@ -0,0 +1,126 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AnalysisPrograms.Download +{ + using System; + using System.ComponentModel.DataAnnotations; + using System.Security.Authentication; + using System.Threading.Tasks; + using AnalysisPrograms.AcousticWorkbench.Orchestration; + using AnalysisPrograms.Production; + using AnalysisPrograms.Production.Arguments; + using global::AcousticWorkbench; + using McMaster.Extensions.CommandLineUtils; + using Spectre.Console; + + public abstract class RemoteRepositoryBase : SubCommandBase + { + protected const string RepositoriesHint = $@" +For a list of valid repositories, use the repositories command: +> {Acoustics.Shared.Meta.Name} {DownloadCommand.DownloadCommandName} {RepositoriesCommand.RepositoriesCommandName} +"; + + [Option( + CommandOptionType.SingleValue, + Description = "Which repository to use to download audio from", + ShortName = "repo")] + [Required] + public string Repository { get; set; } + + [Option( + CommandOptionType.SingleValue, + Description = "Your personal access token for the repository")] + public string AuthToken { get; set; } + + protected Repository ResolvedRepository { get; set; } + + protected IAuthenticatedApi Api { get; set; } + + protected IAnsiConsole Console { get; } = AnsiConsole.Console; + + protected void ValidateRepository() + { + if (this.Repository.IsNullOrEmpty()) + { + throw new CommandLineArgumentException("The repository option must be specified"); + } + + this.ResolvedRepository = Repositories.Find(this.Repository); + if (this.ResolvedRepository == null) + { + this.Console.ErrorLine($"Could not find the repository named `{this.Repository}`"); + this.Console.WriteLine(RepositoriesHint); + throw new ValidationException("The repository must exist"); + } + } + + protected void ValidateAuthToken() + { + if (this.AuthToken.IsNullOrEmpty()) + { + this.Console.InfoLine($"An authentication token is needed to connect to {this.Repository}"); + this.AuthToken = null; + this.AuthToken = this.Console.Prompt( + new TextPrompt("Enter your token:") + .Validate(token => + { + if (token.IsNotWhitespace()) + { + return Spectre.Console.ValidationResult.Success(); + } + + return Spectre.Console.ValidationResult.Error("Cannot be empty"); + })); + } + } + + protected void ShowOptions() + { + Grid grid = new Grid() + .AddColumn(new GridColumn().NoWrap().PadRight(4)) + .AddColumn(); + + this.AddOptionToShowOptions(grid); + + grid.AddRow(nameof(this.Repository), this.Repository.ToString()); + + this.Console.Write(new Panel(grid).Header("Using Options")); + this.Console.WriteLine("\n\n"); + } + + protected abstract void AddOptionToShowOptions(Grid grid); + + protected async Task SignIn() + { + + var status = this.Console + .Status() + .Spinner(Spinner.Known.Aesthetic) + .SpinnerStyle(Style.Plain); + + await status.StartAsync("Logging in", this.LogIn); + } + + private async Task LogIn(StatusContext status) + { + var auth = new AuthenticationService(this.ResolvedRepository.Api); + try + { + var task = auth.CheckLogin(this.AuthToken); + var result = await task.TimeoutAfter(Service.ClientTimeout); + status.Status("Successfully logged in"); + this.Console.SuccessLine($"Logged in as `{result.Username}`"); + + this.Api = result; + } + catch (AuthenticationException aex) + { + status.Status("Authentication failed"); + + throw; + } + } + } +} \ No newline at end of file diff --git a/src/AnalysisPrograms/Download/RepositoriesCommand.cs b/src/AnalysisPrograms/Download/RepositoriesCommand.cs new file mode 100644 index 000000000..3eca70d9d --- /dev/null +++ b/src/AnalysisPrograms/Download/RepositoriesCommand.cs @@ -0,0 +1,38 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AnalysisPrograms.Download +{ + using System; + using System.Threading.Tasks; + using AnalysisPrograms.AcousticWorkbench.Orchestration; + using AnalysisPrograms.Production.Arguments; + using McMaster.Extensions.CommandLineUtils; + using Spectre.Console; + + [Command(RepositoriesCommandName, Description = "Lists available repositories which we can download from")] + public class RepositoriesCommand : SubCommandBase + { + public const string RepositoriesCommandName = "repositories"; + + public override Task Execute(CommandLineApplication app) + { + LoggedConsole.WriteLine("Available repositories:"); + + var table = new Table(); + table.AddColumn(nameof(Repository.Name)); + table.AddColumn(nameof(Repository.HomeUri)); + + foreach (var repository in Repositories.Known) + { + table.AddRow(repository.Name, repository.HomeUri); + } + + AnsiConsole.Write(table); + + return this.Ok(); + } + } + +} \ No newline at end of file diff --git a/src/AnalysisPrograms/Download/SearchCommand.cs b/src/AnalysisPrograms/Download/SearchCommand.cs new file mode 100644 index 000000000..f3f3a86f2 --- /dev/null +++ b/src/AnalysisPrograms/Download/SearchCommand.cs @@ -0,0 +1,197 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AnalysisPrograms.Download +{ + using System; + using System.Collections.Generic; + using System.ComponentModel.DataAnnotations; + using System.IO; + using System.Linq; + using System.Threading.Tasks; + using Acoustics.Shared; + using AnalysisPrograms.Production; + using global::AcousticWorkbench; + using global::AcousticWorkbench.Models; + using McMaster.Extensions.CommandLineUtils; + using Spectre.Console; + + [Command(SearchCommandName, Description = "Preview which files would be downloaded by the batch command")] + public class SearchCommand : DownloadBaseCommand + { + private const string SearchCommandName = "search"; + + public SearchCommand() + { + this.Output = Directory.GetCurrentDirectory().ToDirectoryInfo(); + } + + [Option( + CommandOptionType.MultipleValue, + Description = "Project IDs to filter recordings by")] + public ulong[] ProjectIds { get; set; } + + [Option( + CommandOptionType.MultipleValue, + Description = "Region IDs to filter recordings by")] + public ulong[] RegionIds { get; set; } + + [Option( + CommandOptionType.MultipleValue, + Description = "Site IDs to filter recordings by")] + public ulong[] SiteIds { get; set; } + + [Option( + CommandOptionType.SingleValue, + Description = "A date (inclusive) to filter out recordings. Can parse an ISO8601 date.", + ShortName = "")] + public DateTimeOffset? Start { get; set; } + + [Option( + CommandOptionType.SingleValue, + Description = "A date (exclusive) to filter out recordings. Can parse an ISO8601 date.", + ShortName = "")] + public DateTimeOffset? End { get; set; } + + public override async Task Execute(CommandLineApplication app) + { + this.ValidateBatchOptions(); + + this.ShowOptions(); + + await this.SignIn(); + + this.Console.WriteLine("\n"); + var progress = this.Console.Progress() + .HideCompleted(false) + .Columns( + new TaskDescriptionColumn(), + new ProgressBarColumn(), + new PercentageColumn()); + + var pages = await progress.StartAsync(this.FetchPages); + + this.PrintResults(pages); + + return ExceptionLookup.Ok; + } + + protected void ValidateBatchOptions() + { + this.ValidateCommon(); + + if (this.Start.HasValue ^ this.End.HasValue) + { + throw new InvalidStartOrEndException( + $"If {nameof(this.Start)} or {nameof(this.End)} is specified, then both must be specified"); + } + + var setCount = new[] + { + this.ProjectIds.IsNullOrEmpty(), + this.RegionIds.IsNullOrEmpty(), + this.SiteIds.IsNullOrEmpty(), + }.Count(x => !x); + + if (setCount == 0) + { + throw new ValidationException( + "You must choose to filter by one of" + + $"{nameof(this.ProjectIds)}, {nameof(this.RegionIds)}, or {nameof(this.SiteIds)}"); + } + + if (setCount > 1) + { + throw new ValidationException("Filtering by more than one type of ID is currently not supported"); + } + } + + protected override void AddOptionToShowOptions(Grid grid) + { + MaybeAdd(nameof(this.ProjectIds), this.ProjectIds); + MaybeAdd(nameof(this.RegionIds), this.RegionIds); + MaybeAdd(nameof(this.SiteIds), this.SiteIds); + + grid.AddRow(nameof(this.Start), this.Start?.ToString("o")); + grid.AddRow(nameof(this.End), this.End?.ToString("o")); + + base.AddOptionToShowOptions(grid); + + void MaybeAdd(string name, ulong[] ids) + { + if (ids.IsNullOrEmpty()) + { + return; + } + + grid.AddRow(name, ids.Join(", ")); + } + } + + protected override Task> PageQuery(AudioRecordingService service, int page) + { + Interval? range = this.Start switch + { + null => null, + _ => new(this.Start.Value.UtcDateTime, this.End.Value.UtcDateTime), + }; + + return service.FilterRecordingsForDownload( + range: range, + projectIds: this.ProjectIds, + regionIds: this.RegionIds, + siteIds: this.SiteIds, + page: page); + } + + private async Task[]> FetchPages(ProgressContext progress) + { + var first = await this.GetPage(progress); + + var last = await this.GetPage(progress, this.MaxPage); + + return new[] { first, last }; + } + + private void PrintResults(IReadOnlyCollection[] pages) + { + var table = new Table(); + + table.AddColumns("Id"); + table.AddColumns("Recorded Date"); + table.AddColumns("Site Name"); + table.AddColumns("Url"); + + AddRows(pages.First()); + + table.AddEmptyRow(); + + AddRows(pages.Last()); + + this.Console.MarkupLine("Showing first and last pages:"); + this.Console.Write(table); + + Grid grid = new Grid() + .AddColumn(new GridColumn().NoWrap().PadRight(4)) + .AddColumn() + .AddRow("Total files", $"[lime]{this.Total}[/]"); + + this.Console.Write(new Panel(grid).Header("Summary")); + + void AddRows(IReadOnlyCollection page) + { + foreach (var recording in page) + { + var view = this.ResolvedRepository.Website.GetAudioRecordingViewUri(recording.Id); + table.AddRow( + recording.Id.ToString(), + recording.RecordedDate.ToString("O"), + recording.SiteName, + $"[blue]{view}[/]"); + } + } + } + } + +} \ No newline at end of file diff --git a/src/AnalysisPrograms/MainEntry.cs b/src/AnalysisPrograms/MainEntry.cs index 1c1caca28..3291b2e9a 100644 --- a/src/AnalysisPrograms/MainEntry.cs +++ b/src/AnalysisPrograms/MainEntry.cs @@ -13,7 +13,9 @@ namespace AnalysisPrograms using System.Threading.Tasks; using Acoustics.Shared.Logging; using AnalysisPrograms.Production.Arguments; + using AnalysisPrograms.Production.Spectre.Console; using log4net; + using Spectre.Console; using static System.Environment; /// @@ -35,6 +37,9 @@ public static async Task Main(string[] args) VerbosityToLevel(ApDefaultLogVerbosity ?? LogVerbosity.Info), quietConsole: false); + // ensure spectre console fancy-ness is logged + AnsiConsole.Console = new LoggedAnsiConsole(); + Copyright(); PrepareForErrors(); diff --git a/src/AnalysisPrograms/Production/Arguments/MainArgs.cs b/src/AnalysisPrograms/Production/Arguments/MainArgs.cs index 0c6d9acf5..29d56985b 100644 --- a/src/AnalysisPrograms/Production/Arguments/MainArgs.cs +++ b/src/AnalysisPrograms/Production/Arguments/MainArgs.cs @@ -4,15 +4,16 @@ namespace AnalysisPrograms.Production.Arguments { - using System.Threading.Tasks; using Acoustics.Shared; using AnalysisPrograms.AnalyseLongRecordings; + using AnalysisPrograms.Download; using AnalysisPrograms.Draw.RibbonPlots; using AnalysisPrograms.Draw.Zooming; using AnalysisPrograms.EventStatistics; using AnalysisPrograms.Recognizers.Base; using AnalysisPrograms.SpectrogramGenerator; using McMaster.Extensions.CommandLineUtils; + using System.Threading.Tasks; [Command( Meta.Name, @@ -24,6 +25,7 @@ namespace AnalysisPrograms.Production.Arguments [Subcommand( typeof(HelpArgs), typeof(ListArgs), + typeof(DownloadCommand), typeof(AnalysesAvailable), typeof(CheckEnvironment.Arguments), typeof(AnalyseLongRecording.Arguments), diff --git a/src/AnalysisPrograms/Production/Arguments/SubCommandBase.cs b/src/AnalysisPrograms/Production/Arguments/SubCommandBase.cs index e63de7d64..a8c6389c3 100644 --- a/src/AnalysisPrograms/Production/Arguments/SubCommandBase.cs +++ b/src/AnalysisPrograms/Production/Arguments/SubCommandBase.cs @@ -4,6 +4,7 @@ namespace AnalysisPrograms.Production.Arguments { + using System; using System.ComponentModel.DataAnnotations; using System.Threading.Tasks; using McMaster.Extensions.CommandLineUtils; @@ -15,7 +16,7 @@ public abstract class SubCommandBase /// Gets or sets the Parent command. /// This is set by CommandLineUtils automatically. /// - public MainArgs Parent { get; set; } + public object Parent { get; set; } /// /// This method is called when we run the command. @@ -23,7 +24,19 @@ public abstract class SubCommandBase /// public Task OnExecuteAsync(CommandLineApplication app) { - MainEntry.BeforeExecute(this.Parent, app); + var parent = this.Parent; + while (parent as MainArgs is null) + { + if (this.Parent is SubCommandBase subCommand) + { + parent = subCommand.Parent; + continue; + } + + throw new InvalidOperationException("Cannot find main args"); + } + + MainEntry.BeforeExecute((MainArgs)parent, app); return this.Execute(app); } diff --git a/src/AnalysisPrograms/Production/CommandLineApplicationExtensions.cs b/src/AnalysisPrograms/Production/CommandLineApplicationExtensions.cs index 69de15818..7a7623bb2 100644 --- a/src/AnalysisPrograms/Production/CommandLineApplicationExtensions.cs +++ b/src/AnalysisPrograms/Production/CommandLineApplicationExtensions.cs @@ -4,6 +4,7 @@ namespace AnalysisPrograms.Production { + using System.Collections.Generic; using McMaster.Extensions.CommandLineUtils; public static class CommandLineApplicationExtensions @@ -18,5 +19,20 @@ public static CommandLineApplication Root(this CommandLineApplication app) return root; } + + public static IEnumerable AllCommandsRecursive(this CommandLineApplication app) + { + yield return app; + + foreach (var command in app.Commands) + { + // it'd be nicer if we could just return the enumerable here, but we can't + // so iterate on the result manually + foreach (var result in command.AllCommandsRecursive()) + { + yield return result; + } + } + } } } \ No newline at end of file diff --git a/src/AnalysisPrograms/Production/CustomHelpTextGenerator.cs b/src/AnalysisPrograms/Production/CustomHelpTextGenerator.cs index 10b18dcd7..aecea469c 100644 --- a/src/AnalysisPrograms/Production/CustomHelpTextGenerator.cs +++ b/src/AnalysisPrograms/Production/CustomHelpTextGenerator.cs @@ -7,14 +7,13 @@ namespace AnalysisPrograms.Production { + using McMaster.Extensions.CommandLineUtils; + using McMaster.Extensions.CommandLineUtils.HelpText; using System; using System.Collections.Generic; using System.IO; using System.Linq; - using McMaster.Extensions.CommandLineUtils; - using McMaster.Extensions.CommandLineUtils.HelpText; - /// /// A default implementation of help text generation. /// @@ -22,6 +21,7 @@ public class CustomHelpTextGenerator : DefaultHelpTextGenerator { public CustomHelpTextGenerator() { + this.SortCommandsByName = false; } public Dictionary EnvironmentOptions { get; set; } = null; @@ -62,6 +62,13 @@ protected override void GenerateHeader(CommandLineApplication application, TextW base.GenerateHeader(application, output); } + public void GenerateHeaderOnly( + CommandLineApplication application, + TextWriter output) + { + this.GenerateHeader(application, output); + } + private string FormatEnvironmentVariables() { var result = string.Empty; diff --git a/src/AnalysisPrograms/Production/Exceptions.cs b/src/AnalysisPrograms/Production/Exceptions.cs index 8f2302ea6..0681dd5dc 100644 --- a/src/AnalysisPrograms/Production/Exceptions.cs +++ b/src/AnalysisPrograms/Production/Exceptions.cs @@ -10,6 +10,7 @@ namespace AnalysisPrograms.Production using System.ComponentModel.DataAnnotations; using System.IO; using System.Reflection; + using System.Security.Authentication; using System.Text; using Acoustics.Shared; using Acoustics.Shared.ConfigFile; @@ -34,6 +35,10 @@ public static class ExceptionLookup public static int NoData => 10; + public static int Error => 1; + + public static int AuthenticationError => 99; + internal static Dictionary ErrorLevels => levels ?? (levels = CreateExceptionMap()); public static string FormatReflectionTypeLoadException(Exception exception, bool verbose = false) @@ -55,8 +60,7 @@ public static string FormatReflectionTypeLoadException(Exception exception, bool continue; } - string fusionLog = null; - + string fusionLog; switch (inner) { case FileNotFoundException fnfex: @@ -97,6 +101,10 @@ private static Dictionary CreateExceptionMap() typeof(CommandParsingException), new ExceptionStyle() { ErrorCode = 4 } }, + { + typeof(UnrecognizedCommandParsingException), + new ExceptionStyle() { ErrorCode = 4 } + }, { typeof(DirectoryNotFoundException), @@ -109,6 +117,10 @@ private static Dictionary CreateExceptionMap() // disabled print usage because these exceptions happen at all levels of the stack new ExceptionStyle { ErrorCode = 52, PrintUsage = false } }, + { + typeof(AuthenticationException), + new ExceptionStyle { ErrorCode = AuthenticationError, PrintUsage = false } + }, { typeof(InvalidDurationException), new ExceptionStyle { ErrorCode = 100 } @@ -258,6 +270,8 @@ public class AnalysisOptionDevilException : Exception { } + + public class NoDeveloperMethodException : Exception { private const string StandardMessage = "There is no Developer (Dev) method available for this analysis"; diff --git a/src/AnalysisPrograms/Production/MainEntryUtilities.cs b/src/AnalysisPrograms/Production/MainEntryUtilities.cs index 36eb024c7..ad5b0380e 100644 --- a/src/AnalysisPrograms/Production/MainEntryUtilities.cs +++ b/src/AnalysisPrograms/Production/MainEntryUtilities.cs @@ -22,7 +22,6 @@ namespace AnalysisPrograms using AnalysisPrograms.Production; using AnalysisPrograms.Production.Arguments; using AnalysisPrograms.Production.Parsers; - using log4net; using log4net.Core; using McMaster.Extensions.CommandLineUtils; @@ -69,6 +68,7 @@ internal enum Usages { All, Single, + Node, ListAvailable, NoAction, } @@ -324,8 +324,9 @@ internal static void PrintUsage(string message, Usages usageStyle, string comman } else { - command = root.Commands.FirstOrDefault(x => - x.Name.Equals(commandName, StringComparison.InvariantCultureIgnoreCase)); + command = root + .AllCommandsRecursive() + .FirstOrDefault(x => x.Name.Equals(commandName, StringComparison.InvariantCultureIgnoreCase)); // sometimes this is called from AppDomainUnhandledException, in which case throwing another exception // just gets squashed! @@ -340,16 +341,30 @@ internal static void PrintUsage(string message, Usages usageStyle, string comman command.ShowHelp(false); } + else if (usageStyle == Usages.Node) + { + var node = root + .AllCommandsRecursive() + .FirstOrDefault(x => x.Name.Equals(commandName, StringComparison.InvariantCultureIgnoreCase)); + + var commands = node.Commands; + + using var sb = new StringWriter(); + + ((CustomHelpTextGenerator)node.HelpTextGenerator).GenerateHeaderOnly(node, sb); + ((CustomHelpTextGenerator)CommandLineApplication.HelpTextGenerator).FormatCommands(sb, commands); + + LoggedConsole.WriteLine(sb.ToString()); + } else if (usageStyle == Usages.ListAvailable) { var commands = root.Commands; - using (var sb = new StringWriter()) - { - ((CustomHelpTextGenerator)CommandLineApplication.HelpTextGenerator).FormatCommands(sb, commands); + using var sb = new StringWriter(); - LoggedConsole.WriteLine(sb.ToString()); - } + ((CustomHelpTextGenerator)CommandLineApplication.HelpTextGenerator).FormatCommands(sb, commands); + + LoggedConsole.WriteLine(sb.ToString()); } else if (usageStyle == Usages.NoAction) { diff --git a/src/AnalysisPrograms/Production/Spectre.Console/FileSize.cs b/src/AnalysisPrograms/Production/Spectre.Console/FileSize.cs new file mode 100644 index 000000000..2ac702ec4 --- /dev/null +++ b/src/AnalysisPrograms/Production/Spectre.Console/FileSize.cs @@ -0,0 +1,109 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace Spectre.Console +{ + using System; + using System.Globalization; + + internal enum FileSizeUnit + { + Byte = 0, + KiloByte = 1, + MegaByte = 2, + GigaByte = 3, + TeraByte = 4, + PetaByte = 5, + ExaByte = 6, + ZettaByte = 7, + YottaByte = 8, + } + + // https://raw.githubusercontent.com/spectreconsole/spectre.console/a690ce49556615fea49e61972646eb52a11bbdb5/src/Spectre.Console/Internal/FileSize.cs + internal struct FileSize + { + public double Bytes { get; } + + public FileSizeUnit Unit { get; } + + public string Suffix => this.GetSuffix(); + + public FileSize(double bytes) + { + this.Bytes = bytes; + this.Unit = Detect(bytes); + } + + public FileSize(double bytes, FileSizeUnit unit) + { + this.Bytes = bytes; + this.Unit = unit; + } + + public string Format(CultureInfo? culture = null) + { + var @base = GetBase(this.Unit); + if (@base == 0) + { + @base = 1; + } + + var bytes = this.Bytes / @base; + + return this.Unit == FileSizeUnit.Byte + ? ((int)bytes).ToString(culture ?? CultureInfo.InvariantCulture) + : bytes.ToString("F1", culture ?? CultureInfo.InvariantCulture); + } + + public override string ToString() + { + return this.ToString(suffix: true, CultureInfo.InvariantCulture); + } + + public string ToString(bool suffix = true, CultureInfo? culture = null) + { + if (suffix) + { + return $"{this.Format(culture)} {this.Suffix}"; + } + + return this.Format(culture); + } + + private string GetSuffix() + { + return (this.Bytes, this.Unit) switch + { + (_, FileSizeUnit.KiloByte) => "KB", + (_, FileSizeUnit.MegaByte) => "MB", + (_, FileSizeUnit.GigaByte) => "GB", + (_, FileSizeUnit.TeraByte) => "TB", + (_, FileSizeUnit.PetaByte) => "PB", + (_, FileSizeUnit.ExaByte) => "EB", + (_, FileSizeUnit.ZettaByte) => "ZB", + (_, FileSizeUnit.YottaByte) => "YB", + (1, _) => "byte", + (_, _) => "bytes", + }; + } + + private static FileSizeUnit Detect(double bytes) + { + foreach (var unit in (FileSizeUnit[])Enum.GetValues(typeof(FileSizeUnit))) + { + if (bytes < (GetBase(unit) * 1024)) + { + return unit; + } + } + + return FileSizeUnit.Byte; + } + + private static double GetBase(FileSizeUnit unit) + { + return Math.Pow(1024, (int)unit); + } + } +} diff --git a/src/AnalysisPrograms/Production/Spectre.Console/IAnsiConsoleExtensions.cs b/src/AnalysisPrograms/Production/Spectre.Console/IAnsiConsoleExtensions.cs new file mode 100644 index 000000000..3ad6fd358 --- /dev/null +++ b/src/AnalysisPrograms/Production/Spectre.Console/IAnsiConsoleExtensions.cs @@ -0,0 +1,34 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace Spectre.Console +{ + public static class IAnsiConsoleExtensions + { + private static readonly Style Success = new(foreground: Color.Lime); + private static readonly Style Info = new(foreground: Color.Aqua); + private static readonly Style Error = new(foreground: Color.Red); + private static readonly Style Warn = new(foreground: Color.Yellow); + + public static void SuccessLine(this IAnsiConsole console, string message) + { + console.WriteLine(message, Success); + } + + public static void ErrorLine(this IAnsiConsole console, string message) + { + console.WriteLine(message, Error); + } + + public static void WarnLine(this IAnsiConsole console, string message) + { + console.WriteLine(message, Warn); + } + + public static void InfoLine(this IAnsiConsole console, string message) + { + console.WriteLine(message, Info); + } + } +} diff --git a/src/AnalysisPrograms/Production/Spectre.Console/LoggedAnsiConsole.cs b/src/AnalysisPrograms/Production/Spectre.Console/LoggedAnsiConsole.cs new file mode 100644 index 000000000..2afd0e3d3 --- /dev/null +++ b/src/AnalysisPrograms/Production/Spectre.Console/LoggedAnsiConsole.cs @@ -0,0 +1,47 @@ +namespace AnalysisPrograms.Production.Spectre.Console +{ + using Acoustics.Shared.Logging; + using global::Spectre.Console; + using global::Spectre.Console.Rendering; + using log4net; + + public class LoggedAnsiConsole : IAnsiConsole + { + private static readonly ILog Log = LogManager.Exists(Logging.RootNamespace, Logging.LogFileOnly); + + private readonly IAnsiConsole console; + + public LoggedAnsiConsole() + { + this.console = AnsiConsole.Console; + } + + public Profile Profile => this.console.Profile; + + public IAnsiConsoleCursor Cursor => this.console.Cursor; + + public IAnsiConsoleInput Input => this.console.Input; + + public IExclusivityMode ExclusivityMode => this.console.ExclusivityMode; + + public RenderPipeline Pipeline => this.console.Pipeline; + + public void Clear(bool home) => this.console.Clear(home); + + public void Write(IRenderable renderable) + { + // double render without ansi to log + foreach (var segment in renderable.GetSegments(this)) + { + if (segment.IsControlCode) + { + continue; + } + + Log.Info(segment.Text); + } + + this.console.Write(renderable); + } + } +} diff --git a/src/AudioAnalysisTools/EventStatistics/EventStatistics.cs b/src/AudioAnalysisTools/EventStatistics/EventStatistics.cs index 74443f0b4..e906542a2 100644 --- a/src/AudioAnalysisTools/EventStatistics/EventStatistics.cs +++ b/src/AudioAnalysisTools/EventStatistics/EventStatistics.cs @@ -36,22 +36,6 @@ public EventStatistics() public long? AudioRecordingId { get; set; } - public string ListenUrl - { - get - { - if (this.AudioRecordingId.HasValue) - { - return Api.Default.GetListenUri( - this.AudioRecordingId.Value, - Math.Floor(this.ResultStartSeconds)) - .ToString(); - } - - return string.Empty; - } - } - public DateTimeOffset? AudioRecordingRecordedDate { get; set; } // Note: EventStartSeconds is in base class diff --git a/src/AudioAnalysisTools/Events/Interfaces/ISpectralBand.cs b/src/AudioAnalysisTools/Events/Interfaces/ISpectralBand.cs index d7a6a2bef..1365c792b 100644 --- a/src/AudioAnalysisTools/Events/Interfaces/ISpectralBand.cs +++ b/src/AudioAnalysisTools/Events/Interfaces/ISpectralBand.cs @@ -1,4 +1,4 @@ -// +// // All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). // diff --git a/src/AudioAnalysisTools/Events/Types/HarmonicEvent.cs b/src/AudioAnalysisTools/Events/Types/HarmonicEvent.cs index d10ecab54..68d94c704 100644 --- a/src/AudioAnalysisTools/Events/Types/HarmonicEvent.cs +++ b/src/AudioAnalysisTools/Events/Types/HarmonicEvent.cs @@ -4,7 +4,6 @@ namespace AudioAnalysisTools { - using System; using AudioAnalysisTools.Events; using AudioAnalysisTools.Events.Drawing; using SixLabors.ImageSharp.Processing; @@ -16,7 +15,7 @@ public class HarmonicEvent : SpectralEvent /// /// /// The calculated interval between formant peaks, in hertz. - /// public double HarmonicInterval { get; set; } public override void Draw(IImageProcessingContext graphics, EventRenderingOptions options) diff --git a/tests/Acoustics.Test/AudioAnalysisTools/Events/BlobEventTest.cs b/tests/Acoustics.Test/AudioAnalysisTools/Events/BlobEventTest.cs index 5c8ebc6c9..6ebf5c1ef 100644 --- a/tests/Acoustics.Test/AudioAnalysisTools/Events/BlobEventTest.cs +++ b/tests/Acoustics.Test/AudioAnalysisTools/Events/BlobEventTest.cs @@ -1,4 +1,4 @@ -// +// // All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). // diff --git a/tests/Acoustics.Test/AudioAnalysisTools/HarmonicAnalysis/HarmonicAlgorithmTests.cs b/tests/Acoustics.Test/AudioAnalysisTools/HarmonicAnalysis/HarmonicAlgorithmTests.cs index 4e19f04c9..f667c2a97 100644 --- a/tests/Acoustics.Test/AudioAnalysisTools/HarmonicAnalysis/HarmonicAlgorithmTests.cs +++ b/tests/Acoustics.Test/AudioAnalysisTools/HarmonicAnalysis/HarmonicAlgorithmTests.cs @@ -1,4 +1,4 @@ -// +// // All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). //